Copy changeset replication diffs to S3

cookbooks/planet/files/default/replication-bin/replicate-changesets

@@ -207,6 +207,10 @@ class Replicator
     @config["data_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
   end
 
+  def s3_stem
+    @config["s3_dir"] + format("/%03d/%03d/%03d", sequence / 1000000, (sequence / 1000) % 1000, (sequence % 1000))
+  end
+
   def write_tmp_files!(changesets)
     data_file = data_stem + ".osm.gz"
     tmp_state = @config["state_file"] + ".tmp"
@@ -255,6 +259,14 @@ class Replicator
     # from the dirent and has been updated to account for any allocations.
     fdirsync(File.dirname(data_file))
     fdirsync(File.dirname(@config["state_file"]))
+
+    if @config["s3_dir"]
+      s3_file = s3_stem + ".osm.gz"
+      s3_state_file = s3_stem + ".state.txt"
+
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_file, s3_file)
+      system("/opt/awscli/v2/current/bin/aws", "--profile=osm-pds-upload", "s3", "cp", "--storage-class=INTELLIGENT_TIERING", data_state_file, s3_state_file)
+    end
   end
 
   # saves new state (including the changeset dump xml)

cookbooks/planet/recipes/replication.rb

@@ -208,6 +208,7 @@
   exec_start "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
   sandbox :enable_network => true
   read_write_paths [
+    "/home/planet/.aws",
     "/run/replication",
     "/store/planet/replication/changesets"
   ]

cookbooks/planet/templates/default/changesets.conf.erb

@@ -1,4 +1,5 @@
 state_file: /store/planet/replication/changesets/state.yaml
 db: host=<%= node[:web][:database_host] %> dbname=openstreetmap user=planetdiff password=<%= @password %>
 data_dir: /store/planet/replication/changesets
+s3_dir: s3://osm-planet-eu-central-1/changesets/replication/minute
 lock_file: /run/replication/changesets.lock