Merge pull request #132 from camicroscope/develop

For 3.9.10

.github/workflows/container.yml

@@ -0,0 +1,46 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Container Publish
+
+on:
+  push:
+    branches: ['master', 'develop']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -7,7 +7,7 @@ RUN npm install
 ARG viewer
 ARG fork
 RUN git clone https://github.com/${fork:-camicroscope}/camicroscope.git --branch=${viewer:-master}
-EXPOSE 8010
+EXPOSE 4010
 
 RUN chgrp -R 0 /src && \
     chmod -R g+rwX /src

README.md

@@ -34,6 +34,7 @@ All possible configuration variables are listed in `.env.example`. Renaming the
 |IIP_PATH | IIP server location | http://ca-iip |
 |MONGO_URI | mongo connection uri | mongodb://localhost |
 |MONGO_DB | mongo db to use, default camic |
+|RUN_INDEXER | add indexes and defaults for mongo, default true |
 |GENERATE_KEY_IF_MISSING | automatic generate key in server in not found | false |
 |ENABLE_SECURITY_AT| time at which to enable security; [see parsable times](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/parse)| (not active) |
 

caracal.js

@@ -29,6 +29,7 @@ if (!DISABLE_TF) {
   const Model = require('./handlers/modelTrainer.js');
 }
 
+
 const {connector} = require("./service/database/connector");
 
 var WORKERS = process.env.NUM_THREADS || 4;
@@ -39,6 +40,8 @@ var MONGO_URI = process.env.MONGO_URI || 'mongodb://localhost';
 
 var DISABLE_CSP = process.env.DISABLE_CSP || false;
 
+var RUN_INDEXER = process.env.RUN_INDEXER || true;
+
 const app = express();
 app.use(cookieParser());
 
@@ -194,38 +197,58 @@ app.use(function(err, req, res, next) {
   res.status(statusCode).json(err);
 });
 
-var startApp = function(app) {
-  return function() {
-    // Prepare for SSL/HTTPS
-    var httpsOptions = {};
-    try {
-      var sslPkPath = "./ssl/privatekey.pem";
-      var sslCertPath = "./ssl/certificate.pem";
-      if (fs.existsSync(sslPkPath) && fs.existsSync(sslCertPath)) {
-        console.info("Starting in HTTPS Mode mode");
-        httpsOptions.key = fs.readFileSync(sslPkPath, 'utf8');
-        httpsOptions.cert = fs.readFileSync(sslCertPath, 'utf8');
-      }
-    } catch (err) {
-      console.error(err);
-    }
-    if (httpsOptions.key && httpsOptions.cert) {
-      https.createServer(httpsOptions, app).listen(PORT, () => console.log('listening HTTPS on ' + PORT));
-    } else {
-      app.listen(PORT, () => console.log('listening on ' + PORT));
+function startApp(app) {
+  // Prepare for SSL/HTTPS
+  var httpsOptions = {};
+  try {
+    var sslPkPath = "./ssl/privatekey.pem";
+    var sslCertPath = "./ssl/certificate.pem";
+    if (fs.existsSync(sslPkPath) && fs.existsSync(sslCertPath)) {
+      console.info("Starting in HTTPS Mode mode");
+      httpsOptions.key = fs.readFileSync(sslPkPath, 'utf8');
+      httpsOptions.cert = fs.readFileSync(sslCertPath, 'utf8');
     }
-  };
+  } catch (err) {
+    console.error(err);
+  }
+  if (httpsOptions.key && httpsOptions.cert) {
+    https.createServer(httpsOptions, app).listen(PORT, () => console.log('listening HTTPS on ' + PORT));
+  } else {
+    app.listen(PORT, () => console.log('listening on ' + PORT));
+  }
 };
 
-throng(WORKERS, startApp(app));
+// call this only once no matter what
+function masterHandler() {
+  connector.init().then(() => {
+    const handler = new DataTransformationHandler(MONGO_URI, './json/configuration.json');
+    handler.startHandler();
+  }).then(()=>{
+    if (RUN_INDEXER) {
+      const indexer = require('./idx_mongo.js');
+      indexer.collections();
+      indexer.indexes();
+      indexer.defaults();
+      console.log("added indexes");
+    }
+  }).catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}
+// for each worker
+function workerHandler() {
+  connector.init().then(() => {
+    const handler = new DataTransformationHandler(MONGO_URI, './json/configuration.json');
+    handler.startHandler();
+  }).then(()=>{
+    startApp(app);
+  }).catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}
 
-/** initialize DataTransformationHandler only after database is ready */
-connector.init().then(() => {
-  const handler = new DataTransformationHandler(MONGO_URI, './json/configuration.json');
-  handler.startHandler();
-}).catch((e) => {
-  console.error("error connecting to database");
-  process.exit(1);
-});
+throng({master: masterHandler, start: workerHandler, count: WORKERS});
 
 module.exports = app; // for tests

fixAnnotationLabelIDPatch.js

@@ -0,0 +1,143 @@
+const {MongoClient, ObjectID} = require("mongodb");
+
+function randomId() {
+  // Math.random should be unique because of its seeding algorithm.
+  // Convert it to base 36 (numbers + letters), and grab the first 9 characters
+  // after the decimal.
+  return `_${Math.random()
+      .toString(36)
+      .substr(2, 9)}`;
+}
+//
+class FixAnnotationLabelIDPatch {
+  /**
+   * @constructor to initialize connection to database server
+   */
+  constructor(mongoUrl="mongodb://ca-mongo", db="camic", recordPerBatch=20000) {
+    /** connection specifics */
+    this.mongoUrl = mongoUrl;
+    this.db = db;
+    this.labelMap = new Map();
+    /** connection configurations */
+    const configs = {
+      useUnifiedTopology: true,
+      useNewUrlParser: true,
+    };
+    this.client = new MongoClient(this.mongoUrl, configs);
+
+    // deal batch
+    this.totalBatch;
+    this.totalRecord;
+    this.currentBatch = 1;
+    this.recordPerBatch = recordPerBatch;
+  }
+  async run() {
+    console.log('FixAnnotationLabelIDPatch Start');
+
+    console.log(`waiting to connect ${this.mongoUrl}/${this.db}`);
+    await this.client.connect();
+    console.log(`[database:${this.db}] connected`);
+    // get connector
+    const connector = this.client.db(this.db);
+
+    // config_prelabel
+    const configCollection = connector.collection('configuration');
+    const query = {'config_name': 'preset_label'};
+    const opt = {};
+    const configData = await configCollection.findOne(query, opt);
+
+    // clear the label map
+    this.labelMap.clear();
+    configData.configuration.forEach((label)=>{
+      this.labelMap.set(label.type, label);
+    });
+
+    // preset label annotation without labelId
+    // provenance.analysis.labelId
+    // properties.annotations.labelId
+    const markCollection = connector.collection('mark');
+    this.totalRecord = await markCollection.countDocuments({
+      'provenance.analysis.source': 'human',
+      'provenance.analysis.type': 'label',
+      'provenance.analysis.labelId': {$exists: false},
+    });
+
+    // get total batch
+    this.totalBatch = Math.ceil(this.totalRecord/this.recordPerBatch);
+
+
+    console.log(`TOTAL Annotations Batches: ${this.totalBatch}`);
+
+    while (this.currentBatch <= this.totalBatch) {
+      const isCompleted = await this.runCurrentBatch(markCollection);
+      if (isCompleted) console.log(`Batch ${this.currentBatch} Completed`);
+      this.currentBatch++;
+    }
+
+    console.log('FixAnnotationLabelIDPatch End');
+    return true;
+  }
+
+  async runCurrentBatch(collection) {
+    console.log(`Running ${this.currentBatch} of ${this.totalBatch} Batches`);
+
+    // get current batch marks
+    const marks = await collection.find({
+      'provenance.analysis.source': 'human',
+      'provenance.analysis.type': 'label',
+      'provenance.analysis.labelId': {$exists: false},
+    }, {
+      geometries: 0,
+      skip: 0,
+      limit: this.recordPerBatch,
+    }).toArray();
+
+    for (let idx = 0; idx < marks.length; idx++) {
+      const mark = marks[idx];
+
+      console.log(`(${idx+1}/${marks.length}) => mark Id: ${mark._id}`);
+      const execId = mark.provenance.analysis.execution_id;
+      const executionId = execId.slice(execId.lastIndexOf('_'));
+      const labelType = mark.properties.annotations.notes;
+      // check if the label type on current marks exist on label map
+      var labelId;
+      if (this.labelMap.has(labelType)) {
+        const val = this.labelMap.get(labelType);
+        labelId = val.id;
+      } else {
+        // create a random Id for missing label type
+        labelId = randomId();
+        this.labelMap.set(labelType, {id: labelId});
+      }
+      // update annotation
+      const rs = await collection.updateOne({_id: new ObjectID(mark._id)}, {
+        '$set': {
+          'provenance.analysis.execution_id': executionId,
+          'provenance.analysis.labelId': labelId,
+          'provenance.analysis.name': labelType,
+          'properties.annotations.id': executionId,
+          'properties.annotations.labelId': labelId,
+          'properties.annotations.name': labelType,
+        },
+      });
+      if (rs.result.ok&&rs.result.nModified) {
+        console.log('update success');
+      } else {
+        console.error('update fail');
+      }
+    }
+    return true;
+  }
+}
+async function start() {
+  try {
+    const patch = new FixAnnotationLabelIDPatch();
+    const isCompleted = await patch.run();
+    if (isCompleted) process.exit();
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+start();