Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Diff upload: CTE instead of temporary tables #460

Merged
merged 1 commit into from
Jan 4, 2025
Merged

Diff upload: CTE instead of temporary tables #460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/node_updater.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,20 +66,14 @@ class ApiDB_Node_Updater : public api06::Node_Updater {
bool if_unused;
};

void truncate_temporary_tables();

void replace_old_ids_in_nodes(
std::vector<node_t> &create_nodes,
const std::vector<api06::OSMChange_Tracking::object_id_mapping_t>
&created_node_id_mapping);

void check_unique_placeholder_ids(const std::vector<node_t> &create_nodes);

void insert_new_nodes_to_tmp_table(const std::vector<node_t> &create_nodes);

void copy_tmp_create_nodes_to_current_nodes();

void delete_tmp_create_nodes();
void insert_new_nodes_to_current_table(const std::vector<node_t> &create_nodes);

void lock_current_nodes(const std::vector<osm_nwr_id_t> &ids);

Expand Down
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/relation_updater.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,6 @@ class ApiDB_Relation_Updater : public api06::Relation_Updater {
bool new_member;
};

void truncate_temporary_tables();

/*
* Set id field based on old_id -> id mapping
*
Expand All @@ -101,13 +99,9 @@ class ApiDB_Relation_Updater : public api06::Relation_Updater {

void check_forward_relation_placeholders(const std::vector<relation_t> &create_relations);

void insert_new_relations_to_tmp_table(
void insert_new_relations_to_current_table(
const std::vector<relation_t> &create_relations);

void copy_tmp_create_relations_to_current_relations();

void delete_tmp_create_relations();

void lock_current_relations(const std::vector<osm_nwr_id_t> &ids);

std::vector<std::vector<ApiDB_Relation_Updater::relation_t>>
Expand Down
8 changes: 1 addition & 7 deletions include/cgimap/backend/apidb/changeset_upload/way_updater.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,6 @@ class ApiDB_Way_Updater : public api06::Way_Updater {
bool if_unused;
};

void truncate_temporary_tables();

/*
* Set id field based on old_id -> id mapping
*
Expand All @@ -91,11 +89,7 @@ class ApiDB_Way_Updater : public api06::Way_Updater {

void check_unique_placeholder_ids(const std::vector<way_t> &create_ways);

void insert_new_ways_to_tmp_table(const std::vector<way_t> &create_ways);

void copy_tmp_create_ways_to_current_ways();

void delete_tmp_create_ways();
void insert_new_ways_to_current_table(const std::vector<way_t> &create_ways);

bbox_t calc_way_bbox(const std::vector<osm_nwr_id_t> &ids);

Expand Down
89 changes: 45 additions & 44 deletions src/backend/apidb/changeset_upload/node_updater.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,9 @@ void ApiDB_Node_Updater::process_new_nodes() {

std::vector<osm_nwr_id_t> ids;

truncate_temporary_tables();

check_unique_placeholder_ids(create_nodes);

insert_new_nodes_to_tmp_table(create_nodes);
copy_tmp_create_nodes_to_current_nodes();
delete_tmp_create_nodes();
insert_new_nodes_to_current_table(create_nodes);

// Use new_ids as a result of inserting nodes in tmp table
replace_old_ids_in_nodes(create_nodes, ct.created_node_ids);
Expand Down Expand Up @@ -234,10 +230,6 @@ void ApiDB_Node_Updater::process_delete_nodes() {
delete_nodes.clear();
}

void ApiDB_Node_Updater::truncate_temporary_tables() {
m.exec("TRUNCATE TABLE tmp_create_nodes");
}

/*
* Set id field based on old_id -> id mapping
*
Expand Down Expand Up @@ -278,25 +270,46 @@ void ApiDB_Node_Updater::check_unique_placeholder_ids(
}
}

void ApiDB_Node_Updater::insert_new_nodes_to_tmp_table(
void ApiDB_Node_Updater::insert_new_nodes_to_current_table(
const std::vector<node_t> &create_nodes) {

m.prepare("insert_tmp_create_nodes",
if (create_nodes.empty())
return;

R"(
WITH tmp_node(latitude, longitude, changeset_id, tile, old_id) AS (
SELECT * FROM
UNNEST( CAST($1 as integer[]),
CAST($2 as integer[]),
CAST($3 as bigint[]),
CAST($4 as bigint[]),
CAST($5 as bigint[])
)
// Note: fetching next sequence values has been moved from CTE tmp_nodes
// to a dedicated CTE ids_mapping in order to avoid small gaps in the sequence.
// Postgresql appears to have called nextval() once too often otherwise.

m.prepare("insert_new_nodes_to_current_table", R"(

WITH ids_mapping AS (
SELECT nextval('current_nodes_id_seq'::regclass) AS id,
old_id
FROM
UNNEST($5::bigint[]) AS id(old_id)
),
tmp_nodes AS (
SELECT
UNNEST($1::integer[]) AS latitude,
UNNEST($2::integer[]) AS longitude,
UNNEST($3::bigint[]) AS changeset_id,
true::boolean AS visible,
(now() at time zone 'utc')::timestamp without time zone AS "timestamp",
UNNEST($4::bigint[]) AS tile,
1::bigint AS version,
UNNEST($5::bigint[]) AS old_id
),
insert_op AS (
INSERT INTO current_nodes (id, latitude, longitude, changeset_id,
visible, timestamp, tile, version)
SELECT id, latitude, longitude, changeset_id, visible,
timestamp, tile, version FROM tmp_nodes
INNER JOIN ids_mapping
ON tmp_nodes.old_id = ids_mapping.old_id
)
INSERT INTO tmp_create_nodes (latitude, longitude, changeset_id, tile, old_id)
SELECT * FROM tmp_node
RETURNING id, old_id
)");
SELECT id, old_id
FROM ids_mapping
)");

std::vector<int64_t> lats;
std::vector<int64_t> lons;
Expand All @@ -312,32 +325,20 @@ void ApiDB_Node_Updater::insert_new_nodes_to_tmp_table(
oldids.emplace_back(create_node.old_id);
}

pqxx::result r =
m.exec_prepared("insert_tmp_create_nodes", lats, lons, cs, tiles, oldids);
auto r = m.exec_prepared("insert_new_nodes_to_current_table", lats, lons, cs, tiles, oldids);

if (r.affected_rows() != create_nodes.size())
throw http::server_error(
"Could not create all new nodes in temporary table");
"Could not create all new nodes in current_nodes table");

for (const auto &row : r)
ct.created_node_ids.push_back({ row["old_id"].as<osm_nwr_signed_id_t>(),
row["id"].as<osm_nwr_id_t>(), 1 });
}

void ApiDB_Node_Updater::copy_tmp_create_nodes_to_current_nodes() {
const auto old_id_col(r.column_number("old_id"));
const auto id_col(r.column_number("id"));

auto r = m.exec(
R"(
INSERT INTO current_nodes (id, latitude, longitude, changeset_id,
visible, timestamp, tile, version)
SELECT id, latitude, longitude, changeset_id, visible,
timestamp, tile, version FROM tmp_create_nodes
)");
}

void ApiDB_Node_Updater::delete_tmp_create_nodes() {
for (const auto &row : r) {
ct.created_node_ids.push_back({ row[old_id_col].as<osm_nwr_signed_id_t>(),
row[id_col].as<osm_nwr_id_t>(), 1 });
}

m.exec("DELETE FROM tmp_create_nodes");
}

void ApiDB_Node_Updater::lock_current_nodes(
Expand Down
75 changes: 36 additions & 39 deletions src/backend/apidb/changeset_upload/relation_updater.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,10 @@ void ApiDB_Relation_Updater::delete_relation(osm_changeset_id_t changeset_id,

void ApiDB_Relation_Updater::process_new_relations() {

truncate_temporary_tables();

check_unique_placeholder_ids(create_relations);
check_forward_relation_placeholders(create_relations);

insert_new_relations_to_tmp_table(create_relations);
copy_tmp_create_relations_to_current_relations();
delete_tmp_create_relations();
insert_new_relations_to_current_table(create_relations);

// Use new_ids as a result of inserting nodes/ways in tmp table
replace_old_ids_in_relations(create_relations, ct.created_node_ids,
Expand Down Expand Up @@ -320,10 +316,6 @@ void ApiDB_Relation_Updater::process_delete_relations() {
delete_relations.clear();
}

void ApiDB_Relation_Updater::truncate_temporary_tables() {
m.exec("TRUNCATE TABLE tmp_create_relations");
}

/*
* Set id field based on old_id -> id mapping
*
Expand Down Expand Up @@ -457,20 +449,34 @@ void ApiDB_Relation_Updater::check_forward_relation_placeholders(
}


void ApiDB_Relation_Updater::insert_new_relations_to_tmp_table(
void ApiDB_Relation_Updater::insert_new_relations_to_current_table(
const std::vector<relation_t> &create_relations) {

m.prepare("insert_tmp_create_relations", R"(

WITH tmp_rel (changeset_id, old_id) AS (
SELECT * FROM
UNNEST( CAST($1 AS bigint[]),
CAST($2 AS bigint[])
)
)
INSERT INTO tmp_create_relations (changeset_id, old_id)
SELECT * FROM tmp_rel
RETURNING id, old_id
m.prepare("insert_new_relations_to_current_table", R"(

WITH ids_mapping AS (
SELECT nextval('current_relations_id_seq'::regclass) AS id,
old_id
FROM
UNNEST($2::bigint[]) AS id(old_id)
),
tmp_relations AS (
SELECT
UNNEST($1::bigint[]) AS changeset_id,
true::boolean AS visible,
(now() at time zone 'utc')::timestamp without time zone AS "timestamp",
1::bigint AS version,
UNNEST($2::bigint[]) AS old_id
),
insert_op AS (
INSERT INTO current_relations (id, changeset_id, timestamp, visible, version)
SELECT id, changeset_id, timestamp, visible, version
FROM tmp_relations
INNER JOIN ids_mapping
ON tmp_relations.old_id = ids_mapping.old_id
)
SELECT id, old_id
FROM ids_mapping
)");

std::vector<osm_changeset_id_t> cs;
Expand All @@ -481,30 +487,21 @@ void ApiDB_Relation_Updater::insert_new_relations_to_tmp_table(
oldids.emplace_back(create_relation.old_id);
}

pqxx::result r = m.exec_prepared("insert_tmp_create_relations", cs, oldids);
auto r = m.exec_prepared("insert_new_relations_to_current_table", cs, oldids);

if (r.affected_rows() != create_relations.size())
throw http::server_error(
"Could not create all new relations in temporary table");
"Could not create all new relations in current table");

for (const auto &row : r)
const auto old_id_col(r.column_number("old_id"));
const auto id_col(r.column_number("id"));

for (const auto &row : r) {
ct.created_relation_ids.push_back(
{ row["old_id"].as<osm_nwr_signed_id_t>(), row["id"].as<osm_nwr_id_t>(),
{ row[old_id_col].as<osm_nwr_signed_id_t>(),
row[id_col].as<osm_nwr_id_t>(),
1 });
}

void ApiDB_Relation_Updater::copy_tmp_create_relations_to_current_relations() {

m.exec(
R"(
INSERT INTO current_relations (id, changeset_id, timestamp, visible, version)
SELECT id, changeset_id, timestamp, visible, version FROM tmp_create_relations
)");
}

void ApiDB_Relation_Updater::delete_tmp_create_relations() {

m.exec("DELETE FROM tmp_create_relations");
}
}

void ApiDB_Relation_Updater::lock_current_relations(
Expand Down
Loading
Loading