[{"data":1,"prerenderedAt":1555},["ShallowReactive",2],{"doc:\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fappend-features-to-postgis-table-pyqgis":3},{"id":4,"title":5,"body":6,"description":1544,"extension":1545,"meta":1546,"navigation":273,"path":1551,"seo":1552,"stem":1553,"__hash__":1554},"docs\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fappend-features-to-postgis-table-pyqgis\u002Findex.md","Append Features to a PostGIS Table in PyQGIS",{"type":7,"value":8,"toc":1531},"minimark",[9,13,17,26,194,199,230,234,241,523,544,548,553,711,734,738,745,904,913,970,974,977,1007,1012,1016,1019,1185,1199,1210,1294,1298,1304,1374,1378,1433,1437,1447,1451,1464,1472,1478,1488,1494,1498,1527],[10,11,5],"h1",{"id":12},"append-features-to-a-postgis-table-in-pyqgis",[14,15,16],"p",{},"Reading from a database is easy; writing to one that other people are reading at the same time is where the care goes. A nightly job that appends yesterday's survey points has to match the target schema exactly, has to not double-insert when it is run twice, and above all must not leave the table in a state where half the rows are today's and half are missing — which is precisely what happens when the process dies between a delete and an insert that were not in the same transaction.",[14,18,19,20,25],{},"This recipe belongs to ",[21,22,24],"a",{"href":23},"\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002F","PostGIS and Database Workflows in PyQGIS",". It covers appending through the provider, wrapping multi-step writes in a transaction, mapping fields between mismatched schemas, and batching so memory stays flat on large inputs.",[14,27,28],{},[29,30,35,39,43,50,67,76,83,87,92,99,101,103,111,117,123,127,132,136,139,144,148,151,155,159,162,169,174,177,182,184,187,189],"svg",{"viewBox":31,"role":32,"ariaLabel":33,"xmlns":34},"0 0 760 274","img","Sequence showing a script preparing features, opening a transaction, deleting the previous day's rows, inserting the new ones and committing, with readers seeing only the before or after state","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg",[36,37,38],"title",{},"What a reader sees during a transactional refresh",[40,41,42],"desc",{},"A vertical sequence has three participants: the script, the PostGIS server and a colleague reading the map. The script prepares features outside the transaction, then opens one, deletes yesterday's rows and inserts today's. Throughout that window the colleague's queries still return yesterday's complete data. Only after the commit do they see today's.",[44,45],"rect",{"x":46,"y":46,"width":47,"height":48,"fill":49},"0","760","274","#f6f3ea",[51,52,53],"defs",{},[54,55,62],"marker",{"id":56,"viewBox":57,"refX":58,"refY":59,"markerWidth":60,"markerHeight":60,"orient":61},"pgApArrow","0 0 10 10","8","5","7","auto-start-reverse",[63,64],"path",{"d":65,"fill":66},"M0 0 L10 5 L0 10 z","#2f3b35",[68,69,75],"text",{"x":70,"y":71,"style":72,"fill":73,"textAnchor":74},"380","26","text-anchor:middle;font-size:14px;font-weight:bold;font-family:sans-serif","#17211d","middle","Readers never see the gap between delete and insert",[68,77,82],{"x":78,"y":79,"style":80,"fill":81,"textAnchor":74},"106","58","text-anchor:middle;font-size:12px;font-weight:bold;font-family:sans-serif","#0f766e","your script",[68,84,86],{"x":70,"y":79,"style":80,"fill":85,"textAnchor":74},"#2563eb","PostGIS",[68,88,91],{"x":89,"y":79,"style":80,"fill":90,"textAnchor":74},"646","#b45309","a colleague's map",[93,94],"line",{"x1":78,"y1":95,"x2":78,"y2":96,"stroke":97,"style":98},"68","242","#59645f","stroke-width:1.5;stroke-dasharray:4 4",[93,100],{"x1":70,"y1":95,"x2":70,"y2":96,"stroke":97,"style":98},[93,102],{"x1":89,"y1":95,"x2":89,"y2":96,"stroke":97,"style":98},[44,104],{"x":105,"y":106,"width":107,"height":71,"rx":108,"fill":109,"stroke":97,"style":110},"150","80","188","4","#fffdf7","stroke-width:1.5",[68,112,116],{"x":113,"y":114,"style":115,"fill":66,"textAnchor":74},"244","98","text-anchor:middle;font-size:10px;font-family:sans-serif","BEGIN",[93,118],{"x1":119,"y1":120,"x2":121,"y2":120,"stroke":66,"style":122},"110","93","146","stroke-width:1.5;marker-end:url(#pgApArrow)",[93,124],{"x1":125,"y1":120,"x2":126,"y2":120,"stroke":66,"style":122},"342","376",[44,128],{"x":105,"y":129,"width":107,"height":71,"rx":108,"fill":130,"stroke":131,"style":110},"116","#fdf2e2","#b91c1c",[68,133,135],{"x":113,"y":134,"style":115,"fill":66,"textAnchor":74},"134","DELETE survey_date = yesterday",[93,137],{"x1":119,"y1":138,"x2":121,"y2":138,"stroke":66,"style":122},"129",[44,140],{"x":105,"y":141,"width":107,"height":71,"rx":108,"fill":142,"stroke":143,"style":110},"152","#edf8e9","#15803d",[68,145,147],{"x":113,"y":146,"style":115,"fill":66,"textAnchor":74},"170","INSERT 12 480 rows",[93,149],{"x1":119,"y1":150,"x2":121,"y2":150,"stroke":66,"style":122},"165",[44,152],{"x":105,"y":107,"width":107,"height":71,"rx":108,"fill":153,"stroke":81,"style":154},"#eef7f4","stroke-width:2",[68,156,158],{"x":113,"y":157,"style":115,"fill":66,"textAnchor":74},"206","COMMIT",[93,160],{"x1":119,"y1":161,"x2":121,"y2":161,"stroke":66,"style":122},"201",[44,163],{"x":164,"y":165,"width":166,"height":167,"rx":168,"fill":130,"stroke":90,"style":154},"424","108","196","62","6",[68,170,173],{"x":171,"y":172,"style":115,"fill":66,"textAnchor":74},"522","132","still reads yesterday's",[68,175,176],{"x":171,"y":105,"style":115,"fill":66,"textAnchor":74},"complete dataset",[93,178],{"x1":179,"y1":180,"x2":181,"y2":180,"stroke":90,"style":122},"620","139","642",[44,183],{"x":164,"y":107,"width":166,"height":71,"rx":168,"fill":142,"stroke":143,"style":154},[68,185,186],{"x":171,"y":157,"style":115,"fill":66,"textAnchor":74},"switches to today's, atomically",[93,188],{"x1":179,"y1":161,"x2":181,"y2":161,"stroke":143,"style":122},[68,190,193],{"x":70,"y":191,"style":192,"fill":97,"textAnchor":74},"262","text-anchor:middle;font-size:11px;font-family:sans-serif","Without the transaction there is a window in which the layer is empty for everyone",[195,196,198],"h2",{"id":197},"prerequisites","Prerequisites",[200,201,202,215,227],"ul",{},[203,204,205,209,210,214],"li",{},[206,207,208],"strong",{},"QGIS 3.34 LTR"," or newer with a working PostGIS connection — see ",[21,211,213],{"href":212},"\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fconnect-to-postgis-database-pyqgis\u002F","Connect to a PostGIS Database in PyQGIS",".",[203,216,217,218,222,223,226],{},"A role with ",[219,220,221],"code",{},"INSERT"," (and ",[219,224,225],{},"DELETE",", if you refresh rather than accumulate) on the target table.",[203,228,229],{},"A target table that already exists, with a primary key and a geometry column whose SRID matches what you intend to write.",[195,231,233],{"id":232},"append-through-the-provider","Append through the provider",[14,235,236,237,240],{},"The provider's ",[219,238,239],{},"addFeatures()"," is the direct route: no edit buffer, no undo stack, one round trip per batch.",[242,243,248],"pre",{"className":244,"code":245,"language":246,"meta":247,"style":247},"language-python shiki shiki-themes github-dark","from qgis.core import QgsFeature, QgsVectorLayer\n\ntarget = QgsVectorLayer(uri.uri(False), \"Readings\", \"postgres\")\nprovider = target.dataProvider()\n\nfields = target.fields()\nbatch = []\n\nfor source_feature in incoming.getFeatures():\n    feature = QgsFeature(fields)\n    feature.setGeometry(source_feature.geometry())\n    feature[\"station_id\"] = source_feature[\"station\"]\n    feature[\"reading_m\"] = float(source_feature[\"value\"])\n    feature[\"survey_date\"] = source_feature[\"captured\"]\n    batch.append(feature)\n\nok, written = provider.addFeatures(batch)\nif not ok:\n    raise RuntimeError(provider.error().message())\nprint(f\"inserted {len(written)} rows\")\n","python","",[219,249,250,268,275,307,318,323,334,345,350,365,376,382,405,429,448,454,459,470,482,494],{"__ignoreMap":247},[251,252,254,258,262,265],"span",{"class":93,"line":253},1,[251,255,257],{"class":256},"snl16","from",[251,259,261],{"class":260},"s95oV"," qgis.core ",[251,263,264],{"class":256},"import",[251,266,267],{"class":260}," QgsFeature, QgsVectorLayer\n",[251,269,271],{"class":93,"line":270},2,[251,272,274],{"emptyLinePlaceholder":273},true,"\n",[251,276,278,281,284,287,291,294,298,301,304],{"class":93,"line":277},3,[251,279,280],{"class":260},"target ",[251,282,283],{"class":256},"=",[251,285,286],{"class":260}," QgsVectorLayer(uri.uri(",[251,288,290],{"class":289},"sDLfK","False",[251,292,293],{"class":260},"), ",[251,295,297],{"class":296},"sU2Wk","\"Readings\"",[251,299,300],{"class":260},", ",[251,302,303],{"class":296},"\"postgres\"",[251,305,306],{"class":260},")\n",[251,308,310,313,315],{"class":93,"line":309},4,[251,311,312],{"class":260},"provider ",[251,314,283],{"class":256},[251,316,317],{"class":260}," target.dataProvider()\n",[251,319,321],{"class":93,"line":320},5,[251,322,274],{"emptyLinePlaceholder":273},[251,324,326,329,331],{"class":93,"line":325},6,[251,327,328],{"class":260},"fields ",[251,330,283],{"class":256},[251,332,333],{"class":260}," target.fields()\n",[251,335,337,340,342],{"class":93,"line":336},7,[251,338,339],{"class":260},"batch ",[251,341,283],{"class":256},[251,343,344],{"class":260}," []\n",[251,346,348],{"class":93,"line":347},8,[251,349,274],{"emptyLinePlaceholder":273},[251,351,353,356,359,362],{"class":93,"line":352},9,[251,354,355],{"class":256},"for",[251,357,358],{"class":260}," source_feature ",[251,360,361],{"class":256},"in",[251,363,364],{"class":260}," incoming.getFeatures():\n",[251,366,368,371,373],{"class":93,"line":367},10,[251,369,370],{"class":260},"    feature ",[251,372,283],{"class":256},[251,374,375],{"class":260}," QgsFeature(fields)\n",[251,377,379],{"class":93,"line":378},11,[251,380,381],{"class":260},"    feature.setGeometry(source_feature.geometry())\n",[251,383,385,388,391,394,396,399,402],{"class":93,"line":384},12,[251,386,387],{"class":260},"    feature[",[251,389,390],{"class":296},"\"station_id\"",[251,392,393],{"class":260},"] ",[251,395,283],{"class":256},[251,397,398],{"class":260}," source_feature[",[251,400,401],{"class":296},"\"station\"",[251,403,404],{"class":260},"]\n",[251,406,408,410,413,415,417,420,423,426],{"class":93,"line":407},13,[251,409,387],{"class":260},[251,411,412],{"class":296},"\"reading_m\"",[251,414,393],{"class":260},[251,416,283],{"class":256},[251,418,419],{"class":289}," float",[251,421,422],{"class":260},"(source_feature[",[251,424,425],{"class":296},"\"value\"",[251,427,428],{"class":260},"])\n",[251,430,432,434,437,439,441,443,446],{"class":93,"line":431},14,[251,433,387],{"class":260},[251,435,436],{"class":296},"\"survey_date\"",[251,438,393],{"class":260},[251,440,283],{"class":256},[251,442,398],{"class":260},[251,444,445],{"class":296},"\"captured\"",[251,447,404],{"class":260},[251,449,451],{"class":93,"line":450},15,[251,452,453],{"class":260},"    batch.append(feature)\n",[251,455,457],{"class":93,"line":456},16,[251,458,274],{"emptyLinePlaceholder":273},[251,460,462,465,467],{"class":93,"line":461},17,[251,463,464],{"class":260},"ok, written ",[251,466,283],{"class":256},[251,468,469],{"class":260}," provider.addFeatures(batch)\n",[251,471,473,476,479],{"class":93,"line":472},18,[251,474,475],{"class":256},"if",[251,477,478],{"class":256}," not",[251,480,481],{"class":260}," ok:\n",[251,483,485,488,491],{"class":93,"line":484},19,[251,486,487],{"class":256},"    raise",[251,489,490],{"class":289}," RuntimeError",[251,492,493],{"class":260},"(provider.error().message())\n",[251,495,497,500,503,506,509,512,515,518,521],{"class":93,"line":496},20,[251,498,499],{"class":289},"print",[251,501,502],{"class":260},"(",[251,504,505],{"class":256},"f",[251,507,508],{"class":296},"\"inserted ",[251,510,511],{"class":289},"{len",[251,513,514],{"class":260},"(written)",[251,516,517],{"class":289},"}",[251,519,520],{"class":296}," rows\"",[251,522,306],{"class":260},[14,524,525,528,529,532,533,537,538,540,541,543],{},[206,526,527],{},"Breakdown:"," Constructing ",[219,530,531],{},"QgsFeature(fields)"," from the ",[534,535,536],"em",{},"target"," layer's fields — not the source's — is what keeps the attribute order correct; assigning by name then works regardless of how the source is ordered. Do not set the primary key: leave it unset and let the database's sequence assign it, otherwise two runs collide. ",[219,539,239],{}," returns a success flag and the list of features as written, with their new identifiers filled in, which is useful when a later step needs those ids. A failure returns ",[219,542,290],{}," rather than raising, so the check is not optional.",[195,545,547],{"id":546},"wrap-the-refresh-in-a-transaction","Wrap the refresh in a transaction",[14,549,550,552],{},[219,551,239],{}," on its own is atomic for that one call. A delete-then-insert refresh is two calls, and needs an explicit transaction to be atomic as a pair.",[242,554,556],{"className":244,"code":555,"language":246,"meta":247,"style":247},"from qgis.core import QgsTransaction\n\ntransaction = QgsTransaction.create([target])\nif not transaction.begin():\n    raise RuntimeError(\"could not begin transaction\")\n\ntry:\n    transaction.executeSql(\n        \"DELETE FROM public.readings WHERE survey_date = current_date - 1\",\n        True,\n    )\n    ok, _ = provider.addFeatures(batch)\n    if not ok:\n        raise RuntimeError(provider.error().message())\n    if not transaction.commit():\n        raise RuntimeError(\"commit failed\")\nexcept Exception:\n    transaction.rollback()\n    raise\n",[219,557,558,569,573,583,592,605,609,617,622,630,637,642,651,660,669,678,691,701,706],{"__ignoreMap":247},[251,559,560,562,564,566],{"class":93,"line":253},[251,561,257],{"class":256},[251,563,261],{"class":260},[251,565,264],{"class":256},[251,567,568],{"class":260}," QgsTransaction\n",[251,570,571],{"class":93,"line":270},[251,572,274],{"emptyLinePlaceholder":273},[251,574,575,578,580],{"class":93,"line":277},[251,576,577],{"class":260},"transaction ",[251,579,283],{"class":256},[251,581,582],{"class":260}," QgsTransaction.create([target])\n",[251,584,585,587,589],{"class":93,"line":309},[251,586,475],{"class":256},[251,588,478],{"class":256},[251,590,591],{"class":260}," transaction.begin():\n",[251,593,594,596,598,600,603],{"class":93,"line":320},[251,595,487],{"class":256},[251,597,490],{"class":289},[251,599,502],{"class":260},[251,601,602],{"class":296},"\"could not begin transaction\"",[251,604,306],{"class":260},[251,606,607],{"class":93,"line":325},[251,608,274],{"emptyLinePlaceholder":273},[251,610,611,614],{"class":93,"line":336},[251,612,613],{"class":256},"try",[251,615,616],{"class":260},":\n",[251,618,619],{"class":93,"line":347},[251,620,621],{"class":260},"    transaction.executeSql(\n",[251,623,624,627],{"class":93,"line":352},[251,625,626],{"class":296},"        \"DELETE FROM public.readings WHERE survey_date = current_date - 1\"",[251,628,629],{"class":260},",\n",[251,631,632,635],{"class":93,"line":367},[251,633,634],{"class":289},"        True",[251,636,629],{"class":260},[251,638,639],{"class":93,"line":378},[251,640,641],{"class":260},"    )\n",[251,643,644,647,649],{"class":93,"line":384},[251,645,646],{"class":260},"    ok, _ ",[251,648,283],{"class":256},[251,650,469],{"class":260},[251,652,653,656,658],{"class":93,"line":407},[251,654,655],{"class":256},"    if",[251,657,478],{"class":256},[251,659,481],{"class":260},[251,661,662,665,667],{"class":93,"line":431},[251,663,664],{"class":256},"        raise",[251,666,490],{"class":289},[251,668,493],{"class":260},[251,670,671,673,675],{"class":93,"line":450},[251,672,655],{"class":256},[251,674,478],{"class":256},[251,676,677],{"class":260}," transaction.commit():\n",[251,679,680,682,684,686,689],{"class":93,"line":456},[251,681,664],{"class":256},[251,683,490],{"class":289},[251,685,502],{"class":260},[251,687,688],{"class":296},"\"commit failed\"",[251,690,306],{"class":260},[251,692,693,696,699],{"class":93,"line":461},[251,694,695],{"class":256},"except",[251,697,698],{"class":289}," Exception",[251,700,616],{"class":260},[251,702,703],{"class":93,"line":472},[251,704,705],{"class":260},"    transaction.rollback()\n",[251,707,708],{"class":93,"line":484},[251,709,710],{"class":256},"    raise\n",[14,712,713,715,716,719,720,723,724,726,727,729,730,214],{},[206,714,527],{}," ",[219,717,718],{},"QgsTransaction.create()"," takes the layers that should share one database transaction — passing several keeps a multi-table write consistent. The second argument to ",[219,721,722],{},"executeSql()"," marks the statement as a modification so it participates in the transaction's dirty state. The ",[219,725,613],{},"\u002F",[219,728,695],{}," that rolls back and re-raises is the whole point: without it a failed insert leaves the delete committed and the table short of a day's data. Re-raising rather than swallowing means a scheduled runner sees a non-zero exit, which is what triggers the alert described in ",[21,731,733],{"href":732},"\u002Fpyqgis-fundamentals-environment-setup\u002Fheadless-qgis-and-server-automation\u002Fhandle-errors-and-logging-in-unattended-scripts\u002F","handling errors in unattended scripts",[195,735,737],{"id":736},"commit-in-batches-on-large-inputs","Commit in batches on large inputs",[14,739,740,741,744],{},"Building a list of two million ",[219,742,743],{},"QgsFeature"," objects will exhaust memory long before the insert starts. Flush periodically instead.",[242,746,748],{"className":244,"code":747,"language":246,"meta":247,"style":247},"BATCH = 5000\nbatch = []\ntotal = 0\n\nfor source_feature in incoming.getFeatures():\n    batch.append(build_feature(source_feature, fields))\n    if len(batch) >= BATCH:\n        ok, written = provider.addFeatures(batch)\n        if not ok:\n            raise RuntimeError(provider.error().message())\n        total += len(written)\n        batch.clear()\n\nif batch:\n    ok, written = provider.addFeatures(batch)\n    total += len(written)\nprint(\"appended\", total)\n",[219,749,750,761,769,779,783,793,798,816,825,834,843,856,861,865,872,881,892],{"__ignoreMap":247},[251,751,752,755,758],{"class":93,"line":253},[251,753,754],{"class":289},"BATCH",[251,756,757],{"class":256}," =",[251,759,760],{"class":289}," 5000\n",[251,762,763,765,767],{"class":93,"line":270},[251,764,339],{"class":260},[251,766,283],{"class":256},[251,768,344],{"class":260},[251,770,771,774,776],{"class":93,"line":277},[251,772,773],{"class":260},"total ",[251,775,283],{"class":256},[251,777,778],{"class":289}," 0\n",[251,780,781],{"class":93,"line":309},[251,782,274],{"emptyLinePlaceholder":273},[251,784,785,787,789,791],{"class":93,"line":320},[251,786,355],{"class":256},[251,788,358],{"class":260},[251,790,361],{"class":256},[251,792,364],{"class":260},[251,794,795],{"class":93,"line":325},[251,796,797],{"class":260},"    batch.append(build_feature(source_feature, fields))\n",[251,799,800,802,805,808,811,814],{"class":93,"line":336},[251,801,655],{"class":256},[251,803,804],{"class":289}," len",[251,806,807],{"class":260},"(batch) ",[251,809,810],{"class":256},">=",[251,812,813],{"class":289}," BATCH",[251,815,616],{"class":260},[251,817,818,821,823],{"class":93,"line":347},[251,819,820],{"class":260},"        ok, written ",[251,822,283],{"class":256},[251,824,469],{"class":260},[251,826,827,830,832],{"class":93,"line":352},[251,828,829],{"class":256},"        if",[251,831,478],{"class":256},[251,833,481],{"class":260},[251,835,836,839,841],{"class":93,"line":367},[251,837,838],{"class":256},"            raise",[251,840,490],{"class":289},[251,842,493],{"class":260},[251,844,845,848,851,853],{"class":93,"line":378},[251,846,847],{"class":260},"        total ",[251,849,850],{"class":256},"+=",[251,852,804],{"class":289},[251,854,855],{"class":260},"(written)\n",[251,857,858],{"class":93,"line":384},[251,859,860],{"class":260},"        batch.clear()\n",[251,862,863],{"class":93,"line":407},[251,864,274],{"emptyLinePlaceholder":273},[251,866,867,869],{"class":93,"line":431},[251,868,475],{"class":256},[251,870,871],{"class":260}," batch:\n",[251,873,874,877,879],{"class":93,"line":450},[251,875,876],{"class":260},"    ok, written ",[251,878,283],{"class":256},[251,880,469],{"class":260},[251,882,883,886,888,890],{"class":93,"line":456},[251,884,885],{"class":260},"    total ",[251,887,850],{"class":256},[251,889,804],{"class":289},[251,891,855],{"class":260},[251,893,894,896,898,901],{"class":93,"line":461},[251,895,499],{"class":289},[251,897,502],{"class":260},[251,899,900],{"class":296},"\"appended\"",[251,902,903],{"class":260},", total)\n",[14,905,906,908,909,912],{},[206,907,527],{}," Five thousand features per call is a good default: large enough that per-statement overhead disappears, small enough that memory stays flat and a failure is easy to locate. ",[219,910,911],{},"batch.clear()"," rather than rebinding to a new list keeps the same object and avoids surprising a reference held elsewhere. Note the trailing flush — the loop leaves a partial batch behind whenever the count is not an exact multiple, and forgetting it silently drops up to 4 999 rows.",[14,914,915],{},[29,916,919,922,925,927,930,934,937,942,947,952,958,961,966],{"viewBox":917,"role":32,"ariaLabel":918,"xmlns":34},"0 0 760 244","Memory profile of accumulating every feature before inserting compared with flushing every five thousand, where the batched version stays flat",[36,920,921],{},"Memory during an append: accumulate versus flush",[40,923,924],{},"Two curves over the same run. Accumulating every feature before a single insert climbs steadily to a peak that risks exhausting memory. Flushing every five thousand features produces a low sawtooth that never rises above a small constant.",[44,926],{"x":46,"y":46,"width":47,"height":113,"fill":49},[68,928,929],{"x":70,"y":71,"style":72,"fill":73,"textAnchor":74},"Flushing turns a climbing curve into a flat one",[93,931],{"x1":932,"y1":166,"x2":933,"y2":166,"stroke":97,"style":110},"70","726",[93,935],{"x1":932,"y1":936,"x2":932,"y2":166,"stroke":97,"style":110},"52",[68,938,941],{"x":939,"y":940,"style":115,"fill":97,"textAnchor":74},"46","60","RAM",[68,943,946],{"x":944,"y":945,"style":115,"fill":97,"textAnchor":74},"398","216","features processed",[63,948],{"d":949,"fill":950,"stroke":131,"style":951},"M70 194 L200 168 L330 132 L460 100 L590 72 L718 56","none","stroke-width:2.5",[68,953,957],{"x":954,"y":955,"style":956,"fill":131},"600","96","font-size:11px;font-weight:bold;font-family:sans-serif","accumulate everything",[63,959],{"d":960,"fill":950,"stroke":143,"style":951},"M70 194 L96 176 L96 194 L122 176 L122 194 L148 176 L148 194 L174 176 L174 194 L200 176 L200 194 L226 176 L226 194 L252 176 L252 194 L278 176 L278 194 L304 176 L304 194 L330 176 L330 194 L356 176 L356 194 L382 176 L382 194 L408 176 L408 194 L434 176 L434 194 L460 176 L460 194 L486 176 L486 194 L512 176 L512 194 L538 176 L538 194 L564 176 L564 194 L590 176 L590 194 L616 176 L616 194 L642 176 L642 194 L668 176 L668 194 L694 176 L694 194 L718 178",[68,962,965],{"x":963,"y":964,"style":956,"fill":143},"200","160","flush every 5 000",[68,967,969],{"x":70,"y":968,"style":192,"fill":97,"textAnchor":74},"234","The sawtooth is the batch filling and being released after each insert",[195,971,973],{"id":972},"make-the-job-safe-to-run-twice","Make the job safe to run twice",[14,975,976],{},"Scheduled jobs get re-run: by a retry, by an operator, by a colleague who did not know it had already gone. Give the table a natural key and let the database refuse the duplicate.",[242,978,980],{"className":244,"code":979,"language":246,"meta":247,"style":247},"connection.executeSql(\"\"\"\n    ALTER TABLE public.readings\n    ADD CONSTRAINT readings_station_date_unique UNIQUE (station_id, survey_date)\n\"\"\")\n",[219,981,982,990,995,1000],{"__ignoreMap":247},[251,983,984,987],{"class":93,"line":253},[251,985,986],{"class":260},"connection.executeSql(",[251,988,989],{"class":296},"\"\"\"\n",[251,991,992],{"class":93,"line":270},[251,993,994],{"class":296},"    ALTER TABLE public.readings\n",[251,996,997],{"class":93,"line":277},[251,998,999],{"class":296},"    ADD CONSTRAINT readings_station_date_unique UNIQUE (station_id, survey_date)\n",[251,1001,1002,1005],{"class":93,"line":309},[251,1003,1004],{"class":296},"\"\"\"",[251,1006,306],{"class":260},[14,1008,1009,1011],{},[206,1010,527],{}," With the constraint in place, a second run fails loudly on the first duplicate instead of quietly doubling every count in every downstream map. Combine it with the delete-then-insert pattern above and re-running becomes genuinely idempotent: yesterday's rows go, today's arrive, and the totals are right whether the job ran once or five times.",[195,1013,1015],{"id":1014},"reconcile-after-the-write","Reconcile after the write",[14,1017,1018],{},"An append that reports success has still only told you that the statements executed. Whether the table now holds what it should is a separate question, and asking it costs one query.",[242,1020,1022],{"className":244,"code":1021,"language":246,"meta":247,"style":247},"expected = incoming.featureCount()\n\nbefore = connection.executeSql(\n    \"SELECT count(*) FROM public.readings WHERE survey_date = current_date - 1\"\n)[0][0]\n\n# … the transactional refresh …\n\nafter = connection.executeSql(\n    \"SELECT count(*) FROM public.readings WHERE survey_date = current_date - 1\"\n)[0][0]\n\nlog.info(\"source=%d before=%d after=%d\", expected, before, after)\nif after != expected:\n    raise RuntimeError(f\"expected {expected} rows for yesterday, found {after}\")\n",[219,1023,1024,1034,1038,1048,1053,1067,1071,1077,1081,1090,1094,1106,1110,1137,1150],{"__ignoreMap":247},[251,1025,1026,1029,1031],{"class":93,"line":253},[251,1027,1028],{"class":260},"expected ",[251,1030,283],{"class":256},[251,1032,1033],{"class":260}," incoming.featureCount()\n",[251,1035,1036],{"class":93,"line":270},[251,1037,274],{"emptyLinePlaceholder":273},[251,1039,1040,1043,1045],{"class":93,"line":277},[251,1041,1042],{"class":260},"before ",[251,1044,283],{"class":256},[251,1046,1047],{"class":260}," connection.executeSql(\n",[251,1049,1050],{"class":93,"line":309},[251,1051,1052],{"class":296},"    \"SELECT count(*) FROM public.readings WHERE survey_date = current_date - 1\"\n",[251,1054,1055,1058,1060,1063,1065],{"class":93,"line":320},[251,1056,1057],{"class":260},")[",[251,1059,46],{"class":289},[251,1061,1062],{"class":260},"][",[251,1064,46],{"class":289},[251,1066,404],{"class":260},[251,1068,1069],{"class":93,"line":325},[251,1070,274],{"emptyLinePlaceholder":273},[251,1072,1073],{"class":93,"line":336},[251,1074,1076],{"class":1075},"sjoCn","# … the transactional refresh …\n",[251,1078,1079],{"class":93,"line":347},[251,1080,274],{"emptyLinePlaceholder":273},[251,1082,1083,1086,1088],{"class":93,"line":352},[251,1084,1085],{"class":260},"after ",[251,1087,283],{"class":256},[251,1089,1047],{"class":260},[251,1091,1092],{"class":93,"line":367},[251,1093,1052],{"class":296},[251,1095,1096,1098,1100,1102,1104],{"class":93,"line":378},[251,1097,1057],{"class":260},[251,1099,46],{"class":289},[251,1101,1062],{"class":260},[251,1103,46],{"class":289},[251,1105,404],{"class":260},[251,1107,1108],{"class":93,"line":384},[251,1109,274],{"emptyLinePlaceholder":273},[251,1111,1112,1115,1118,1121,1124,1126,1129,1131,1134],{"class":93,"line":407},[251,1113,1114],{"class":260},"log.info(",[251,1116,1117],{"class":296},"\"source=",[251,1119,1120],{"class":289},"%d",[251,1122,1123],{"class":296}," before=",[251,1125,1120],{"class":289},[251,1127,1128],{"class":296}," after=",[251,1130,1120],{"class":289},[251,1132,1133],{"class":296},"\"",[251,1135,1136],{"class":260},", expected, before, after)\n",[251,1138,1139,1141,1144,1147],{"class":93,"line":431},[251,1140,475],{"class":256},[251,1142,1143],{"class":260}," after ",[251,1145,1146],{"class":256},"!=",[251,1148,1149],{"class":260}," expected:\n",[251,1151,1152,1154,1156,1158,1160,1163,1166,1169,1171,1174,1176,1179,1181,1183],{"class":93,"line":450},[251,1153,487],{"class":256},[251,1155,490],{"class":289},[251,1157,502],{"class":260},[251,1159,505],{"class":256},[251,1161,1162],{"class":296},"\"expected ",[251,1164,1165],{"class":289},"{",[251,1167,1168],{"class":260},"expected",[251,1170,517],{"class":289},[251,1172,1173],{"class":296}," rows for yesterday, found ",[251,1175,1165],{"class":289},[251,1177,1178],{"class":260},"after",[251,1180,517],{"class":289},[251,1182,1133],{"class":296},[251,1184,306],{"class":260},[14,1186,1187,1189,1190,1192,1193,1195,1196,214],{},[206,1188,527],{}," Counting the same slice before and after gives three numbers that together describe what happened: how many rows arrived, how many were already there, and how many remain. A mismatch between ",[219,1191,1168],{}," and ",[219,1194,1178],{}," catches the failures that no exception reports — a source feed that truncated, a filter that excluded more than intended, a partial batch that was never flushed. Raising on the mismatch converts a wrong result into a failed job, which is the behaviour a scheduler can act on, following the plausibility discipline in ",[21,1197,1198],{"href":732},"Handle Errors and Logging in Unattended Scripts",[14,1200,1201,1202,1205,1206,1209],{},"Two further checks are worth running periodically rather than nightly: ",[219,1203,1204],{},"SELECT count(*) FROM readings WHERE geom IS NULL"," finds rows that arrived without geometry, and ",[219,1207,1208],{},"SELECT ST_SRID(geom), count(*) FROM readings GROUP BY 1"," proves that every row really carries the SRID the column claims. Both are cheap enough to run weekly and both find problems that no single night's log would reveal.",[14,1211,1212],{},[29,1213,1216,1219,1222,1225,1232,1236,1241,1247,1250,1253,1256,1259,1262,1266,1269,1274,1278,1281,1286,1290],{"viewBox":1214,"role":32,"ariaLabel":1215,"xmlns":34},"0 0 760 246","Three counts taken around a nightly append — source features, rows before, rows after — with a mismatch between source and after marked as the signal that catches silent truncation",[36,1217,1218],{},"Three counts that describe what the job did",[40,1220,1221],{},"The source feature count, the row count for that day before the refresh and the row count after it are logged together. When the source and after counts match, the run is verified. When they diverge, the job raises rather than reporting success, catching a truncated feed that no exception would have reported.",[44,1223],{"x":46,"y":46,"width":47,"height":1224,"fill":49},"246",[51,1226,1227],{},[54,1228,1230],{"id":1229,"viewBox":57,"refX":58,"refY":59,"markerWidth":60,"markerHeight":60,"orient":61},"pgRecArrow",[63,1231],{"d":65,"fill":66},[68,1233,1235],{"x":70,"y":1234,"style":72,"fill":73,"textAnchor":74},"28","Success is a number that matches, not a statement that ran",[44,1237],{"x":1238,"y":1239,"width":963,"height":940,"rx":58,"fill":1240,"stroke":85,"style":154},"20","64","#eff3ff",[68,1242,1246],{"x":1243,"y":1244,"style":1245,"fill":85,"textAnchor":74},"120","88","text-anchor:middle;font-size:11px;font-weight:bold;font-family:sans-serif","source features",[68,1248,1249],{"x":1243,"y":119,"style":192,"fill":66,"textAnchor":74},"12 480",[44,1251],{"x":1252,"y":1239,"width":963,"height":940,"rx":58,"fill":109,"stroke":97,"style":154},"280",[68,1254,1255],{"x":70,"y":1244,"style":1245,"fill":73,"textAnchor":74},"rows before",[68,1257,1258],{"x":70,"y":119,"style":192,"fill":66,"textAnchor":74},"12 480 from the last run",[44,1260],{"x":1261,"y":1239,"width":963,"height":940,"rx":58,"fill":142,"stroke":143,"style":951},"540",[68,1263,1265],{"x":1264,"y":1244,"style":1245,"fill":143,"textAnchor":74},"640","rows after",[68,1267,1268],{"x":1264,"y":119,"style":192,"fill":66,"textAnchor":74},"12 480 — verified",[93,1270],{"x1":1271,"y1":1272,"x2":48,"y2":1272,"stroke":66,"style":1273},"220","94","stroke-width:2;marker-end:url(#pgRecArrow)",[93,1275],{"x1":1276,"y1":1272,"x2":1277,"y2":1272,"stroke":66,"style":1273},"480","534",[44,1279],{"x":1252,"y":141,"width":1280,"height":940,"rx":58,"fill":130,"stroke":131,"style":951},"460",[68,1282,1285],{"x":1283,"y":1284,"style":1245,"fill":131,"textAnchor":74},"510","176","source 41 · after 41 — the feed truncated overnight",[68,1287,1289],{"x":1283,"y":1288,"style":192,"fill":66,"textAnchor":74},"198","counts agree with each other and with nothing useful — compare against before",[93,1291],{"x1":1243,"y1":1292,"x2":48,"y2":1284,"stroke":131,"style":1293},"124","stroke-width:2;stroke-dasharray:5 4;marker-end:url(#pgRecArrow)",[195,1295,1297],{"id":1296},"qgis-version-compatibility","QGIS version compatibility",[14,1299,1300,1301,1303],{},"The examples target ",[206,1302,208],{}," (Python 3.12).",[1305,1306,1307,1323],"table",{},[1308,1309,1310],"thead",{},[1311,1312,1313,1317,1320],"tr",{},[1314,1315,1316],"th",{},"QGIS version",[1314,1318,1319],{},"Python",[1314,1321,1322],{},"Notes",[1324,1325,1326,1343,1353,1364],"tbody",{},[1311,1327,1328,1332,1335],{},[1329,1330,1331],"td",{},"3.22 LTR",[1329,1333,1334],{},"3.9",[1329,1336,1337,1192,1339,1342],{},[219,1338,239],{},[219,1340,1341],{},"QgsTransaction"," behave as shown.",[1311,1344,1345,1348,1350],{},[1329,1346,1347],{},"3.28 LTR",[1329,1349,1334],{},[1329,1351,1352],{},"Adds provider feature flags for fast insert; no code change needed.",[1311,1354,1355,1358,1361],{},[1329,1356,1357],{},"3.34 LTR",[1329,1359,1360],{},"3.12",[1329,1362,1363],{},"Baseline for this page.",[1311,1365,1366,1369,1371],{},[1329,1367,1368],{},"3.40 \u002F 3.44",[1329,1370,1360],{},[1329,1372,1373],{},"Transaction groups can be enabled per project, which changes when edits are flushed in the GUI but not in this script.",[195,1375,1377],{"id":1376},"troubleshooting","Troubleshooting",[200,1379,1380,1386,1400,1410,1421,1427],{},[203,1381,1382,1385],{},[206,1383,1384],{},"\"duplicate key value violates unique constraint\"."," The primary key was set on the incoming features. Leave it unset so the sequence assigns it — or, if this is the natural-key constraint firing, the job has already run.",[203,1387,1388,1391,1392,1395,1396,1399],{},[206,1389,1390],{},"Geometry type mismatch."," The target column is declared as ",[219,1393,1394],{},"MultiPolygon"," and the source features are single polygons. Convert with ",[219,1397,1398],{},"geometry.convertToMultiType()"," before appending.",[203,1401,1402,1405,1406,214],{},[206,1403,1404],{},"SRID mismatch."," PostGIS rejects geometry whose SRID differs from the column constraint. Reproject before writing — the transformation itself is covered in ",[21,1407,1409],{"href":1408},"\u002Fspatial-data-processing-automation\u002Fcoordinate-reference-systems\u002Ftransform-point-coordinates-pyqgis\u002F","Transform Point Coordinates in PyQGIS",[203,1411,1412,715,1415,1417,1418,1420],{},[206,1413,1414],{},"Nothing was inserted and no error appeared.",[219,1416,239],{}," returned ",[219,1419,290],{}," and the result was not checked. Always test it.",[203,1422,1423,1426],{},[206,1424,1425],{},"The insert is extremely slow."," Every row is being committed separately, or a trigger fires per row. Batch the calls, and consider dropping non-essential indexes for a very large bulk load and rebuilding them afterwards.",[203,1428,1429,1432],{},[206,1430,1431],{},"Attribute values are all null."," The features were built from the source layer's fields rather than the target's, so assignment by name wrote into positions the target does not have.",[195,1434,1436],{"id":1435},"conclusion","Conclusion",[14,1438,1439,1440,1443,1444,1446],{},"Appending to PostGIS is ",[219,1441,1442],{},"provider.addFeatures()"," with features built from the target's fields, the primary key left alone, and the result checked. Anything that deletes as well as inserts belongs inside a ",[219,1445,1341],{}," with a rollback on failure; anything large belongs in batches with a trailing flush; and anything scheduled deserves a unique constraint so a second run fails loudly rather than doubling the data.",[195,1448,1450],{"id":1449},"frequently-asked-questions","Frequently Asked Questions",[14,1452,1453,1456,1459,1460,1463],{},[206,1454,1455],{},"Should I use the layer's edit buffer instead?",[219,1457,1458],{},"startEditing()"," \u002F ",[219,1461,1462],{},"commitChanges()"," builds an undo stack in memory and is the right choice inside an interactive plugin. For a bulk load, going straight to the provider avoids that overhead entirely.",[14,1465,1466,1469,1471],{},[206,1467,1468],{},"How do I get the identifiers the database assigned?",[219,1470,239],{}," returns the written features with their new ids populated, provided the provider can read them back — which it can for a PostGIS table with a serial primary key.",[14,1473,1474,1477],{},[206,1475,1476],{},"Can I append to a query layer?","\nNo. Query layers are read-only. Append to the underlying table and let the query pick the rows up.",[14,1479,1480,1483,1484,214],{},[206,1481,1482],{},"What happens if the source has fields the target lacks?","\nThey are ignored when you build features from the target's field list. Adding the column to the target first is the deliberate alternative — see ",[21,1485,1487],{"href":1486},"\u002Fspatial-data-processing-automation\u002Fattribute-tables-and-field-management\u002Fadd-field-to-layer-pyqgis\u002F","Add a Field to a Layer in PyQGIS",[14,1489,1490,1493],{},[206,1491,1492],{},"Does a transaction lock the table for readers?","\nNo. PostgreSQL readers are never blocked by a writer; they continue to see the pre-commit snapshot. Two concurrent writers touching the same rows will block each other, which is the behaviour you want.",[195,1495,1497],{"id":1496},"related","Related",[200,1499,1500,1505,1509,1515,1521],{},[203,1501,1502,1504],{},[21,1503,24],{"href":23}," — the guide this recipe belongs to",[203,1506,1507],{},[21,1508,213],{"href":212},[203,1510,1511],{},[21,1512,1514],{"href":1513},"\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fwrite-vector-layer-to-geopackage-pyqgis\u002F","Write a Vector Layer to GeoPackage in PyQGIS",[203,1516,1517],{},[21,1518,1520],{"href":1519},"\u002Fspatial-data-processing-automation\u002Fattribute-tables-and-field-management\u002Fupdate-attribute-values-pyqgis\u002F","Update Attribute Values in PyQGIS",[203,1522,1523],{},[21,1524,1526],{"href":1525},"\u002Fpyqgis-fundamentals-environment-setup\u002Fheadless-qgis-and-server-automation\u002Fschedule-pyqgis-scripts-with-cron\u002F","Schedule PyQGIS Scripts with cron",[1528,1529,1530],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sjoCn, html code.shiki .sjoCn{--shiki-default:#9AA79F}",{"title":247,"searchDepth":270,"depth":270,"links":1532},[1533,1534,1535,1536,1537,1538,1539,1540,1541,1542,1543],{"id":197,"depth":270,"text":198},{"id":232,"depth":270,"text":233},{"id":546,"depth":270,"text":547},{"id":736,"depth":270,"text":737},{"id":972,"depth":270,"text":973},{"id":1014,"depth":270,"text":1015},{"id":1296,"depth":270,"text":1297},{"id":1376,"depth":270,"text":1377},{"id":1435,"depth":270,"text":1436},{"id":1449,"depth":270,"text":1450},{"id":1496,"depth":270,"text":1497},"Insert new features into an existing PostGIS table from PyQGIS, wrap the write in a transaction, match the target schema, and commit in batches so a nightly job never leaves the table half-written.","md",{"slug":1547,"type":1548,"breadcrumb":1549,"datePublished":1550,"dateModified":1550},"append-features-to-postgis-table-pyqgis","article","Append to PostGIS","2026-08-10","\u002Fspatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fappend-features-to-postgis-table-pyqgis",{"title":5,"description":1544},"spatial-data-processing-automation\u002Fpostgis-and-database-workflows\u002Fappend-features-to-postgis-table-pyqgis\u002Findex","paOMrmWUdtO1JOKOifZ6thHpIEWlKmw9MviRJXaghlk",1786401338503]