Acctcga

.github/workflows/preview_new_data.yml

@@ -0,0 +1,79 @@
+on:
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  changed_files:
+    runs-on: ubuntu-latest
+    outputs:
+      new_study_dir: ${{ steps.new-dir.outputs.NEW_DIR_LOCATION }}
+      new_study_name: ${{ steps.new-study.outputs.NEW_STUDY_NAME }}
+    steps:
+      - name: Get New Directories Added
+        id: changed-files-dir-names
+        uses: tj-actions/changed-files@v38
+        with:
+          dir_names: "true"
+      - name: Extract Directory Name of New Study
+        id: new-dir
+        shell: bash
+        run: |
+          echo "NEW_DIR_LOCATION=$( echo ${{ steps.changed-files-dir-names.outputs.added_files }} | awk '{print $1}' )" >> $GITHUB_OUTPUT
+
+      - name: Extract New Study Name
+        id: new-study
+        shell: bash
+        run: |
+          echo "NEW_STUDY_NAME=$( cut -d "/" -f2- <<< ${{ steps.new-dir.outputs.NEW_DIR_LOCATION }} )" >> $GITHUB_OUTPUT
+  preview:
+    needs: changed_files
+    runs-on: ubuntu-latest
+    container: docker.io/okteto/okteto:2.19.1
+    steps:
+      - name: Checkout Datahub Repository
+        uses: actions/checkout@v3
+
+      - name: Copy Files to Study Directory
+        shell: bash
+        run: |
+          # mkdir -p study/${{ needs.changed_files.outputs.new_study_name }}
+          cp -v -R ${{ needs.changed_files.outputs.new_study_dir }} preview_infrastructure/cbioportal-docker-compose/study/${{ needs.changed_files.outputs.new_study_name }}
+
+      - name: Context
+        uses: okteto/context@latest
+        with:
+          url: ${{secrets.OKTETO_URL}}
+          token: ${{ secrets.OKTETO_TOKEN }}
+
+      - name: Okteto Build to Import Studies
+        working-directory: preview_infrastructure/cbioportal-docker-compose
+        run: |
+          okteto build --no-cache -t okteto.dev/datahub-cbioportal:okteto-with-volume-mounts cbioportal
+
+      - name: Deploy preview environment
+        uses: okteto/deploy-preview@latest
+        env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          name: pr-${{ github.event.number }}-averyniceday
+          file: preview_infrastructure/cbioportal-docker-compose/okteto.yml
+          timeout: 15m
+
+      - name: Wait For Response
+        uses: nev7n/[email protected]
+        with:
+          url: 'https://cbioportal-pr-${{ github.event.number }}-averyniceday.cloud.okteto.net/'
+          responseCode: 200
+          timeout: 600000 # 10 minutes
+          interval: 30000 # 30 seconds
+
+      - name: "Activate Namespace"
+        uses: okteto/namespace@latest
+        with:
+          namespace: pr-${{github.event.number}}-averyniceday
+
+      - name: Kubectl test
+        run: |
+          okteto kubeconfig
+          kubectl exec -it deployment/cbioportal -- metaImport.py -u http://localhost:8080 -s study/${{needs.changed_files.outputs.new_study_name}}/ -o

README.md

@@ -3,7 +3,7 @@ The datahub is a repository for data storage only. It contains staging files whi
 
 Behind the scenes git-lfs is used to manage the large files. https://github.com/github/git-lfs
 
-## Test Status
+## Test Status   
 Validation status of all studies on Datahub master branch. This runs weekly using the validation code from the cBioPortal master branch. It also validates if the studies on cbioportal.org and on Datahub are in sync.
 
 [![CircleCI](https://circleci.com/gh/cBioPortal/datahub/tree/master.svg?style=svg)](https://circleci.com/gh/cBioPortal/datahub/tree/master)

preview_infrastructure/cbioportal-docker-compose/.env

@@ -0,0 +1,3 @@
+DOCKER_IMAGE_CBIOPORTAL=registry.cloud.okteto.net/averyniceday/cbioportal-docker-compose-cbioportal:okteto-with-volume-mounts
+DOCKER_IMAGE_SESSION_SERVICE=cbioportal/session-service:0.6.1
+DOCKER_IMAGE_MYSQL=registry.cloud.okteto.net/averyniceday/cbioportal-docker-compose-cbioportal-database:okteto-with-volume-mounts

preview_infrastructure/cbioportal-docker-compose/config/init.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+VERSION=$(grep DOCKER_IMAGE_CBIOPORTAL ../.env | cut -d '=' -f 2-)
+
+docker run --rm -it $VERSION cat /cbioportal-webapp/WEB-INF/classes/portal.properties | \
+    sed 's/db.host=.*/db.host=cbioportal-database:3306/g' | \
+    sed 's|db.connection_string=.*|db.connection_string=jdbc:mysql://cbioportal-database:3306/|g' \
+> portal.properties

preview_infrastructure/cbioportal-docker-compose/data/init.sh

@@ -0,0 +1,2 @@
+# Download the seed database
+wget -O cgds.sql "https://raw.githubusercontent.com/cBioPortal/cbioportal/v5.3.6/db-scripts/src/main/resources/cgds.sql" && wget -O seed.sql.gz "https://github.com/cBioPortal/datahub/raw/master/seedDB/seedDB_hg19_archive/seed-cbioportal_hg19_v2.12.14.sql.gz"

preview_infrastructure/cbioportal-docker-compose/data/init_hg38.sh

@@ -0,0 +1,3 @@
+# Download the seed database
+
+wget -O cgds.sql "https://raw.githubusercontent.com/cBioPortal/cbioportal/v5.3.6/db-scripts/src/main/resources/cgds.sql" && wget -O seed.sql.gz "https://github.com/cBioPortal/datahub/raw/master/seedDB/seedDB_hg19_hg38_archive/seed-cbioportal_hg19_hg38_v2.12.14.sql.gz"

preview_infrastructure/cbioportal-docker-compose/docker-compose.arm64.yml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  cbioportal-database:
+    image: biarms/mysql:5.7
+  cbioportal-session-database:
+    image: mongo:4.2

preview_infrastructure/cbioportal-docker-compose/docker-compose.override.yml

@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  cbioportal:
+    # Note this is as dev version since the latest v4 images don't work on arm64
+    image: cbioportal/cbioportal:5.2.6
+  cbioportal-database:
+    image: biarms/mysql:5.7
+  cbioportal-session-database:
+    image: mongo:4.2

preview_infrastructure/cbioportal-docker-compose/docker-compose.yml

@@ -0,0 +1,63 @@
+version: '3'
+
+services:
+  cbioportal:
+    restart: unless-stopped
+    image: ${DOCKER_IMAGE_CBIOPORTAL}
+    container_name: cbioportal-container
+    environment:
+      SHOW_DEBUG_INFO: "true"
+    ports:
+      - "8080:8080"
+    volumes:
+     - ./study:/study/
+     - ./config/portal.properties:/cbioportal/portal.properties
+    depends_on:
+     - cbioportal-database
+     - cbioportal-session
+    networks:
+     - cbio-net
+    command: /bin/sh -c "java -Xms2g -Xmx4g -Dauthenticate=noauthsessionservice -Dsession.service.url=http://cbioportal-session:5000/api/sessions/my_portal/ -jar webapp-runner.jar -AmaxHttpHeaderSize=16384 -AconnectionTimeout=20000 --enable-compression /cbioportal-webapp"
+  cbioportal-database:
+    restart: unless-stopped
+    image: ${DOCKER_IMAGE_MYSQL}
+    container_name: cbioportal-database-container
+    environment:
+      MYSQL_DATABASE: cbioportal
+      MYSQL_USER: cbio_user
+      MYSQL_PASSWORD: somepassword
+      MYSQL_ROOT_PASSWORD: somepassword
+    volumes:
+     - ./data/cgds.sql:/docker-entrypoint-initdb.d/cgds.sql
+     - ./data/seed.sql.gz:/docker-entrypoint-initdb.d/seed.sql.gz
+     - cbioportal_mysql_data:/var/lib/mysql
+    networks:
+     - cbio-net
+  cbioportal-session:
+    restart: unless-stopped
+    image: ${DOCKER_IMAGE_SESSION_SERVICE}
+    container_name: cbioportal-session-container
+    environment:
+      SERVER_PORT: 5000
+      JAVA_OPTS: -Dspring.data.mongodb.uri=mongodb://cbioportal-session-database:27017/session-service
+    depends_on:
+      - cbioportal-session-database
+    networks:
+      - cbio-net
+  cbioportal-session-database:
+    restart: unless-stopped
+    image: mongo:4.2
+    container_name: cbioportal-session-database-container
+    environment:
+      MONGO_INITDB_DATABASE: session_service
+    volumes:
+      - cbioportal_mongo_data:/data/db
+    networks:
+      - cbio-net
+
+networks:
+  cbio-net: 
+
+volumes:
+  cbioportal_mysql_data:
+  cbioportal_mongo_data:

preview_infrastructure/cbioportal-docker-compose/init.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+for d in config data study; do
+    cd $d; ./init.sh
+    cd ..
+done
+
+# add override docker file for arm64
+# see https://github.com/cBioPortal/cbioportal/issues/9829
+if [[ ! -f "docker-compose.override.yml" ]] && [[ "$(arch)" = "arm64" ]]; then
+    cp docker-compose.arm64.yml docker-compose.override.yml
+fi

preview_infrastructure/cbioportal-docker-compose/okteto.yml

@@ -0,0 +1,7 @@
+deploy:
+  commands: 
+   - apt-get install -y wget
+   - ./init.sh
+   - okteto build --no-cache -t okteto.dev/datahub-cbioportal:okteto-with-volume-mounts cbioportal
+   - okteto kubeconfig
+  compose: docker-compose.yml

preview_infrastructure/cbioportal-docker-compose/study/init.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# download data hub study and import
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+DATAHUB_STUDIES="${DATAHUB_STUDIES:-lgg_ucsf_2014}"
+for study in ${DATAHUB_STUDIES}; do
+        wget -O ${study}.tar.gz "https://cbioportal-datahub.s3.amazonaws.com/${study}.tar.gz"
+        tar xvfz ${study}.tar.gz
+done