diff --git a/.github/actions/push/action.yaml b/.github/actions/push/action.yaml
new file mode 100644
index 000000000..1811ef4cd
--- /dev/null
+++ b/.github/actions/push/action.yaml
@@ -0,0 +1,30 @@
+# copied from internal repo https://github.com/allenai/docker-images
+name: Push image to Beaker
+description: Push image to Beaker images.
+inputs:
+  image:
+    description: The tag of the built image
+    required: true
+  beaker:
+    description: Upload release images to Beaker under this name
+    required: false
+    default: ""
+  latest:
+    description: If the image should also be pushed as latest
+    required: false
+    default: false
+runs:
+  using: composite
+  # note, name used for this is associated with the beaker key in repo secrets
+  steps:
+    - shell: bash
+      if: inputs.beaker != '' # previously startsWith(github.ref, 'refs/tags/') && ...
+      run: |
+        # Push release to Beaker.
+        SHORT_SHA=$(git rev-parse --short HEAD)
+        beaker image create --name "${{ inputs.beaker }}-${SHORT_SHA}-${{ github.run_id }}" ${{ inputs.image }}
+        # We delete the previous version. This doesn't actually delete the backing Docker image, so
+        # we'll still benefit from layer caching when we push new version. The image might not exist
+        # yet, so it's ok if this fails.
+        beaker image delete nathanl/${{ inputs.beaker }} || true
+        beaker image create --name ${{ inputs.beaker }} ${{ inputs.image }}
\ No newline at end of file
diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml
new file mode 100644
index 000000000..957b5bb6b
--- /dev/null
+++ b/.github/actions/setup/action.yaml
@@ -0,0 +1,25 @@
+# copied from internal repo https://github.com/allenai/docker-images
+name: Build setup
+description: Setup the environment for a build
+inputs:
+  beaker_token:
+    description: The Beaker user token
+    required: true
+  # ghcr_token:
+  #   description: GHCR token
+  #   required: true
+  # ghcr_user:
+  #   description: GHCR user name
+  #   required: true
+runs:
+  using: composite
+  steps:
+    - uses: allenai/setup-beaker@v2
+      with:
+        token: ${{ inputs.beaker_token }}
+        workspace: ai2/oe-adapt
+
+    # - shell: bash
+    #   run: |
+    #     # Login to GitHub container registry.
+    #     echo ${{ inputs.ghcr_token }} | docker login ghcr.io -u ${{ inputs.ghcr_user }} --password-stdin
\ No newline at end of file
diff --git a/.github/workflows/push-image-olmo.yml b/.github/workflows/push-image-olmo.yml
new file mode 100644
index 000000000..12c5be539
--- /dev/null
+++ b/.github/workflows/push-image-olmo.yml
@@ -0,0 +1,79 @@
+# This is an example workflow file.
+#
+# When you add a new image, copy this file and then change all mentions of "hello-world" with
+# the name of your new image.
+#
+# Read through the rest of the comments in this file to figure out how it works, and what else
+# you need to change.
+name: build_open_instruct_olmo
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    # Run this workflow anytime a push updates one of the files in the image's directory
+    # (other than the README), and anytime there's a new release tag for this image.
+    paths:
+      - 'open_instruct/**'
+      - '!open_instruct/README.md'
+      - 'requirements-olmo.txt'
+      - 'Dockerfile.olmo'
+      # Note, add .olmo dockerfile + requirements if adding auto build to those
+    branches: [main]
+  pull_request: # note, comment this out for running on every push
+    # Also run on PRs that update the files in the image's directory (other than README).
+    branches: [main]
+    paths:
+      - 'open_instruct/**'
+      - '!open_instruct/README.md'
+      - 'requirements-olmo.txt'
+      - 'Dockerfile.olmo'
+  workflow_dispatch:  # This allows us to manually trigger a build through the GitHub UI.
+
+env:
+  DOCKER_BUILDKIT: "1"
+
+jobs:
+  build:
+    name: open_instruct
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: (github.event_name != 'workflow_run') || (github.event.workflow_run.conclusion == 'success')
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup environment
+        uses: ./.github/actions/setup
+        with:
+          beaker_token: ${{ secrets.BEAKER_TOKEN }}
+          # ghcr_token: ${{ secrets.GHCR_TOKEN }}
+          # ghcr_user: ${{ secrets.GHCR_USER }}
+
+      # big images fail, trying this
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+  
+      - name: Build image
+        run: |
+          docker build \
+              --build-arg BUILDKIT_INLINE_CACHE=1 \
+              --build-arg CUDA=12.1.0 --build-arg \
+              TARGET=cudnn8-devel --build-arg DIST=ubuntu20.04 \
+              --build-arg REQUIRE=requirements.txt \
+              -f Dockerfile.olmo . \
+              -t open_instruct_olmo
+              
+
+      - name: Check image
+        run: |
+          docker run --rm open_instruct_olmo
+
+      - name: Push image
+        # if: github.event_name != 'pull_request'
+        uses: ./.github/actions/push
+        with:
+          image: open_instruct_olmo  # this is the tag of the image we just built in the previous step
+          beaker: open_instruct_olmo_auto  # this is the name of the image on Beaker
+          latest: true  # this flag says we should also push this as the 'latest' version to GHCR
\ No newline at end of file
diff --git a/.github/workflows/push-image.yml b/.github/workflows/push-image.yml
new file mode 100644
index 000000000..4e80a6adc
--- /dev/null
+++ b/.github/workflows/push-image.yml
@@ -0,0 +1,78 @@
+# This is an example workflow file.
+#
+# When you add a new image, copy this file and then change all mentions of "hello-world" with
+# the name of your new image.
+#
+# Read through the rest of the comments in this file to figure out how it works, and what else
+# you need to change.
+name: build_open_instruct
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    # Run this workflow anytime a push updates one of the files in the image's directory
+    # (other than the README), and anytime there's a new release tag for this image.
+    paths:
+      - 'open_instruct/**'
+      - '!open_instruct/README.md'
+      - 'requirements.txt'
+      - 'Dockerfile'
+      # Note, add .olmo dockerfile + requirements if adding auto build to those
+    branches: [main]
+  pull_request: # note, comment this out for running on every push
+    # Also run on PRs that update the files in the image's directory (other than README).
+    branches: [main]
+    paths:
+      - 'open_instruct/**'
+      - '!open_instruct/README.md'
+      - 'requirements.txt'
+      - 'Dockerfile'
+  workflow_dispatch:  # This allows us to manually trigger a build through the GitHub UI.
+
+env:
+  DOCKER_BUILDKIT: "1"
+
+jobs:
+  build:
+    name: open_instruct
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: (github.event_name != 'workflow_run') || (github.event.workflow_run.conclusion == 'success')
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup environment
+        uses: ./.github/actions/setup
+        with:
+          beaker_token: ${{ secrets.BEAKER_TOKEN }}
+          # ghcr_token: ${{ secrets.GHCR_TOKEN }}
+          # ghcr_user: ${{ secrets.GHCR_USER }}
+
+      # big images fail, trying this
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+  
+      - name: Build image
+        run: |
+          docker build \
+              --build-arg BUILDKIT_INLINE_CACHE=1 \
+              --build-arg CUDA=12.1.0 --build-arg \
+              TARGET=cudnn8-devel --build-arg DIST=ubuntu20.04 \
+              --build-arg REQUIRE=requirements.txt . \
+              -t open_instruct
+              
+
+      - name: Check image
+        run: |
+          docker run --rm open_instruct
+
+      - name: Push image
+        # if: github.event_name != 'pull_request'
+        uses: ./.github/actions/push
+        with:
+          image: open_instruct  # this is the tag of the image we just built in the previous step
+          beaker: open_instruct_auto  # this is the name of the image on Beaker
+          latest: true  # this flag says we should also push this as the 'latest' version to GHCR
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 41e517397..a5e225470 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -131,7 +131,7 @@ RUN python -m nltk.downloader punkt
 
 COPY open_instruct open_instruct
 COPY eval eval
-COPY ds_configs ds_configs
+COPY configs configs
 COPY scripts scripts
 RUN chmod +x scripts/*
 
diff --git a/Dockerfile.olmo b/Dockerfile.olmo
index bde04795f..3ea607041 100644
--- a/Dockerfile.olmo
+++ b/Dockerfile.olmo
@@ -134,7 +134,7 @@ RUN python -m nltk.downloader punkt
 
 COPY open_instruct open_instruct
 COPY eval eval
-COPY ds_configs ds_configs
+COPY configs configs
 COPY scripts scripts
 RUN chmod +x scripts/*