Skip to content

Commit

Permalink
Diff upload: CTE instead of temporary tables
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmd-osm
mmd-osm committed Jan 4, 2025
1 parent c2e0270 commit b8b2cb4
Show file tree
Hide file tree
Showing 7 changed files with 119 additions and 184 deletions.
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/node_updater.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,20 +66,14 @@ class ApiDB_Node_Updater : public api06::Node_Updater {
bool if_unused;
};

void truncate_temporary_tables();

void replace_old_ids_in_nodes(
std::vector<node_t> &create_nodes,
const std::vector<api06::OSMChange_Tracking::object_id_mapping_t>
&created_node_id_mapping);

void check_unique_placeholder_ids(const std::vector<node_t> &create_nodes);

void insert_new_nodes_to_tmp_table(const std::vector<node_t> &create_nodes);

void copy_tmp_create_nodes_to_current_nodes();

void delete_tmp_create_nodes();
void insert_new_nodes_to_current_table(const std::vector<node_t> &create_nodes);

void lock_current_nodes(const std::vector<osm_nwr_id_t> &ids);

Expand Down
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/relation_updater.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ class ApiDB_Relation_Updater : public api06::Relation_Updater {
bool new_member;
};

void truncate_temporary_tables();

/*
* Set id field based on old_id -> id mapping
*
Expand All @@ -101,13 +99,9 @@ class ApiDB_Relation_Updater : public api06::Relation_Updater {

void check_forward_relation_placeholders(const std::vector<relation_t> &create_relations);

void insert_new_relations_to_tmp_table(
void insert_new_relations_to_current_table(
const std::vector<relation_t> &create_relations);

void copy_tmp_create_relations_to_current_relations();

void delete_tmp_create_relations();

void lock_current_relations(const std::vector<osm_nwr_id_t> &ids);

std::vector<std::vector<ApiDB_Relation_Updater::relation_t>>
Expand Down
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/way_updater.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,6 @@ class ApiDB_Way_Updater : public api06::Way_Updater {
bool if_unused;
};

void truncate_temporary_tables();

/*
* Set id field based on old_id -> id mapping
*
Expand All @@ -91,11 +89,7 @@ class ApiDB_Way_Updater : public api06::Way_Updater {

void check_unique_placeholder_ids(const std::vector<way_t> &create_ways);

void insert_new_ways_to_tmp_table(const std::vector<way_t> &create_ways);

void copy_tmp_create_ways_to_current_ways();

void delete_tmp_create_ways();
void insert_new_ways_to_current_table(const std::vector<way_t> &create_ways);

bbox_t calc_way_bbox(const std::vector<osm_nwr_id_t> &ids);

Expand Down
89 changes: 45 additions & 44 deletions src/backend/apidb/changeset_upload/node_updater.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,9 @@ void ApiDB_Node_Updater::process_new_nodes() {

std::vector<osm_nwr_id_t> ids;

truncate_temporary_tables();

check_unique_placeholder_ids(create_nodes);

insert_new_nodes_to_tmp_table(create_nodes);
copy_tmp_create_nodes_to_current_nodes();
delete_tmp_create_nodes();
insert_new_nodes_to_current_table(create_nodes);

// Use new_ids as a result of inserting nodes in tmp table
replace_old_ids_in_nodes(create_nodes, ct.created_node_ids);
Expand Down Expand Up @@ -234,10 +230,6 @@ void ApiDB_Node_Updater::process_delete_nodes() {
delete_nodes.clear();
}

void ApiDB_Node_Updater::truncate_temporary_tables() {
m.exec("TRUNCATE TABLE tmp_create_nodes");
}

/*
* Set id field based on old_id -> id mapping
*
Expand Down Expand Up @@ -278,25 +270,46 @@ void ApiDB_Node_Updater::check_unique_placeholder_ids(
}
}

void ApiDB_Node_Updater::insert_new_nodes_to_tmp_table(
void ApiDB_Node_Updater::insert_new_nodes_to_current_table(
const std::vector<node_t> &create_nodes) {

m.prepare("insert_tmp_create_nodes",
if (create_nodes.empty())
return;

R"(
WITH tmp_node(latitude, longitude, changeset_id, tile, old_id) AS (
SELECT * FROM
UNNEST( CAST($1 as integer[]),
CAST($2 as integer[]),
CAST($3 as bigint[]),
CAST($4 as bigint[]),
CAST($5 as bigint[])
)
// Note: fetching next sequence values has been moved from CTE tmp_nodes
// to a dedicated CTE ids_mapping in order to avoid small gaps in the sequence.
// Postgresql appears to have called nextval() once too often otherwise.

m.prepare("insert_new_nodes_to_current_table", R"(
WITH ids_mapping AS (
SELECT nextval('current_nodes_id_seq'::regclass) AS id,
old_id
FROM
UNNEST($5::bigint[]) AS id(old_id)
),
tmp_nodes AS (
SELECT
UNNEST($1::integer[]) AS latitude,
UNNEST($2::integer[]) AS longitude,
UNNEST($3::bigint[]) AS changeset_id,
true::boolean AS visible,
(now() at time zone 'utc')::timestamp without time zone AS "timestamp",
UNNEST($4::bigint[]) AS tile,
1::bigint AS version,
UNNEST($5::bigint[]) AS old_id
),
insert_op AS (
INSERT INTO current_nodes (id, latitude, longitude, changeset_id,
visible, timestamp, tile, version)
SELECT id, latitude, longitude, changeset_id, visible,
timestamp, tile, version FROM tmp_nodes
INNER JOIN ids_mapping
ON tmp_nodes.old_id = ids_mapping.old_id
)
INSERT INTO tmp_create_nodes (latitude, longitude, changeset_id, tile, old_id)
SELECT * FROM tmp_node
RETURNING id, old_id
)");
SELECT id, old_id
FROM ids_mapping
)");

std::vector<int64_t> lats;
std::vector<int64_t> lons;
Expand All @@ -312,32 +325,20 @@ void ApiDB_Node_Updater::insert_new_nodes_to_tmp_table(
oldids.emplace_back(create_node.old_id);
}

pqxx::result r =
m.exec_prepared("insert_tmp_create_nodes", lats, lons, cs, tiles, oldids);
auto r = m.exec_prepared("insert_new_nodes_to_current_table", lats, lons, cs, tiles, oldids);

if (r.affected_rows() != create_nodes.size())
throw http::server_error(
"Could not create all new nodes in temporary table");
"Could not create all new nodes in current_nodes table");

for (const auto &row : r)
ct.created_node_ids.push_back({ row["old_id"].as<osm_nwr_signed_id_t>(),
row["id"].as<osm_nwr_id_t>(), 1 });
}

void ApiDB_Node_Updater::copy_tmp_create_nodes_to_current_nodes() {
const auto old_id_col(r.column_number("old_id"));
const auto id_col(r.column_number("id"));

auto r = m.exec(
R"(
INSERT INTO current_nodes (id, latitude, longitude, changeset_id,
visible, timestamp, tile, version)
SELECT id, latitude, longitude, changeset_id, visible,
timestamp, tile, version FROM tmp_create_nodes
)");
}

void ApiDB_Node_Updater::delete_tmp_create_nodes() {
for (const auto &row : r) {
ct.created_node_ids.push_back({ row[old_id_col].as<osm_nwr_signed_id_t>(),
row[id_col].as<osm_nwr_id_t>(), 1 });
}

m.exec("DELETE FROM tmp_create_nodes");
}

void ApiDB_Node_Updater::lock_current_nodes(
Expand Down
75 changes: 36 additions & 39 deletions src/backend/apidb/changeset_upload/relation_updater.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,10 @@ void ApiDB_Relation_Updater::delete_relation(osm_changeset_id_t changeset_id,

void ApiDB_Relation_Updater::process_new_relations() {

truncate_temporary_tables();

check_unique_placeholder_ids(create_relations);
check_forward_relation_placeholders(create_relations);

insert_new_relations_to_tmp_table(create_relations);
copy_tmp_create_relations_to_current_relations();
delete_tmp_create_relations();
insert_new_relations_to_current_table(create_relations);

// Use new_ids as a result of inserting nodes/ways in tmp table
replace_old_ids_in_relations(create_relations, ct.created_node_ids,
Expand Down Expand Up @@ -320,10 +316,6 @@ void ApiDB_Relation_Updater::process_delete_relations() {
delete_relations.clear();
}

void ApiDB_Relation_Updater::truncate_temporary_tables() {
m.exec("TRUNCATE TABLE tmp_create_relations");
}

/*
* Set id field based on old_id -> id mapping
*
Expand Down Expand Up @@ -457,20 +449,34 @@ void ApiDB_Relation_Updater::check_forward_relation_placeholders(
}


void ApiDB_Relation_Updater::insert_new_relations_to_tmp_table(
void ApiDB_Relation_Updater::insert_new_relations_to_current_table(
const std::vector<relation_t> &create_relations) {

m.prepare("insert_tmp_create_relations", R"(
WITH tmp_rel (changeset_id, old_id) AS (
SELECT * FROM
UNNEST( CAST($1 AS bigint[]),
CAST($2 AS bigint[])
)
)
INSERT INTO tmp_create_relations (changeset_id, old_id)
SELECT * FROM tmp_rel
RETURNING id, old_id
m.prepare("insert_new_relations_to_current_table", R"(
WITH ids_mapping AS (
SELECT nextval('current_relations_id_seq'::regclass) AS id,
old_id
FROM
UNNEST($2::bigint[]) AS id(old_id)
),
tmp_relations AS (
SELECT
UNNEST($1::bigint[]) AS changeset_id,
true::boolean AS visible,
(now() at time zone 'utc')::timestamp without time zone AS "timestamp",
1::bigint AS version,
UNNEST($2::bigint[]) AS old_id
),
insert_op AS (
INSERT INTO current_relations (id, changeset_id, timestamp, visible, version)
SELECT id, changeset_id, timestamp, visible, version
FROM tmp_relations
INNER JOIN ids_mapping
ON tmp_relations.old_id = ids_mapping.old_id
)
SELECT id, old_id
FROM ids_mapping
)");

std::vector<osm_changeset_id_t> cs;
Expand All @@ -481,30 +487,21 @@ void ApiDB_Relation_Updater::insert_new_relations_to_tmp_table(
oldids.emplace_back(create_relation.old_id);
}

pqxx::result r = m.exec_prepared("insert_tmp_create_relations", cs, oldids);
auto r = m.exec_prepared("insert_new_relations_to_current_table", cs, oldids);

if (r.affected_rows() != create_relations.size())
throw http::server_error(
"Could not create all new relations in temporary table");
"Could not create all new relations in current table");

for (const auto &row : r)
const auto old_id_col(r.column_number("old_id"));
const auto id_col(r.column_number("id"));

for (const auto &row : r) {
ct.created_relation_ids.push_back(
{ row["old_id"].as<osm_nwr_signed_id_t>(), row["id"].as<osm_nwr_id_t>(),
{ row[old_id_col].as<osm_nwr_signed_id_t>(),
row[id_col].as<osm_nwr_id_t>(),
1 });
}

void ApiDB_Relation_Updater::copy_tmp_create_relations_to_current_relations() {

m.exec(
R"(
INSERT INTO current_relations (id, changeset_id, timestamp, visible, version)
SELECT id, changeset_id, timestamp, visible, version FROM tmp_create_relations
)");
}

void ApiDB_Relation_Updater::delete_tmp_create_relations() {

m.exec("DELETE FROM tmp_create_relations");
}
}

void ApiDB_Relation_Updater::lock_current_relations(
Expand Down
Loading

0 comments on commit b8b2cb4

Please sign in to comment.