Version 4.3.1 bump

README.md

@@ -60,7 +60,7 @@ to get it is from PyPI using pip, but it can also be built from source (see [BUI
 > pip install minknow_api
 
 # Verify API is installed correctly (from a checkout of this repository):
-> python ./python/examples/list_sequencing_positions.py --host localhost --port 9501
+> python ./python/minknow_api/examples/list_sequencing_positions.py --host localhost --port 9501
 
 # Possible output if running minknow locally:
 #   Available sequencing positions on localhost:9501:
@@ -71,7 +71,7 @@ to get it is from PyPI using pip, but it can also be built from source (see [BUI
 
 The package contains plenty of documentation in its docstrings, although for an overview of the
 MinKNOW APIs themselves you may prefer to read the [API description files](protos/minknow_api/) -
-see below for further discussion of these files. There are also [examples](python/examples/) (like
+see below for further discussion of these files. There are also [examples](python/minknow_api/examples/) (like
 the `list_sequencing_positions.py` script in the above instructions) that show how to perform some
 common tasks.
 
@@ -148,7 +148,7 @@ This can be accessed via the ports reported by `flow_cell_positions` on the mana
 protocols, as well as providing information about the current and previous protocol runs. Note that
 information about protocol runs from before the last restart is not available via this API.
 
-See the [`start_protocol` example](python/examples/start_protocol.py) for an example of how to use
+See the [`start_protocol` example](python/minknow_api/examples/start_protocol.py) for an example of how to use
 this service to start a protocol.
 
 #### acquisition.proto
@@ -257,6 +257,31 @@ channel = grpc.secure_channel("gxb1234:9502",
                               options=(("grpc.ssl_target_name_override", "localhost"),))
 ```
 
+###### Connecting from Manager or Connection classes
+
+The `Manager` or `Connection` classes will use the following strategy to determine the CA
+(certificate authority) file to use to validate TLS connections to MinKNOW:
+
+1. If `MINKNOW_TRUSTED_CA` environment variable is present and contains the path to a file that exists, that file will be used.
+2. If run from MinKNOW's internal `ont-python` environment (eg: as part of a protocol script) the CA
+   file for this installation of MinKNOW will be used.
+3. If MinKNOW is installed on the system in the standard location (see below), the CA file from that
+   installation of MinKNOW will be used.
+
+The default MinKNOW installation path depends on the operating system:
+- Windows: `C:\Program Files\OxfordNanopore\MinKNOW`
+- OSX: `/Applications/MinKNOW.app/Contents/Resources`
+- Linux: `/opt/ont/minknow`
+
+The CA file is located at `conf/rpc-certs/ca.crt` within the MinKNOW installation.
+
+If `MissingMinknowSSlCertError` is raised, then the most likely causes are:
+
+1. Non-default installation path was used
+2. Initiating from a remote connection, so `ca.crt` file is not available
+
+Both cases will require setting `MINKNOW_TRUSTED_CA` environment variable to the correct path. In the case of a remote connection, then the `ca.crt` file will need to be copied to the remote client first.
+
 We are working on improving this experience in future versions of MinKNOW.
 
 ### Glossary

proto/minknow_api/acquisition.proto

@@ -11,7 +11,7 @@ import "google/protobuf/timestamp.proto";
 service AcquisitionService {
     // Starts reading data from the device
     //
-    // Some setup calls will need to be made before starting data acquisition: particularly setting the analysis configuration, 
+    // Some setup calls will need to be made before starting data acquisition: particularly setting the analysis configuration,
     // calibration, read writer and bulk writer config and some device calls such as setting the sampling frequency
     //
     // If acqusition is already running (even in the FINISHING state), this call will fail.
@@ -34,14 +34,14 @@ service AcquisitionService {
     // Watches for status changes within MinKNOW. Status states are defined from MinknowStatus enum.
     // This is a bi-directional stream where the incoming response stream will return everytime the status has changed
     // and the request stream is used to stop the watcher. Refer to http://www.grpc.io/docs/tutorials/basic/python.html
-    // to see how bi-directoional streaming works in grpc, but essentially when calling this function the user will have 
+    // to see how bi-directoional streaming works in grpc, but essentially when calling this function the user will have
     // to pass in a generator that will eventually yield a WatchForStatusChangeRequest(stop=True) to the cpp side.
     // A wrapper class for this is provided in the Python code.
     //
-    // The function will first return with the current status that MinKNOW is in. Every response thereafter will be a 
+    // The function will first return with the current status that MinKNOW is in. Every response thereafter will be a
     // change from one status to another.
     //
-    // The ERROR_STATUS state includes errors during transition between states. If that happens, MinKNOW will 
+    // The ERROR_STATUS state includes errors during transition between states. If that happens, MinKNOW will
     // try to revert to the READY state. It is up to the user to determine if they wish to try to wait for MinKNOW to
     // correct itself or to try some other course of action
     rpc watch_for_status_change (stream WatchForStatusChangeRequest) returns (stream WatchForStatusChangeResponse) {
@@ -98,7 +98,7 @@ service AcquisitionService {
     rpc get_current_acquisition_run (GetCurrentAcquisitionRunRequest) returns (AcquisitionRunInfo) {
         option idempotency_level = NO_SIDE_EFFECTS;
     }
-    
+
     // Specify the signal reader to use
     //
     // Since 3.6
@@ -113,7 +113,7 @@ service AcquisitionService {
 
 enum MinknowStatus {
     // ERROR_STATUS occurs when there is an error during a transition from one state to another.
-    // Minknow will try to revert into the READY state to try to stabalise itself, but there is no guarantee of 
+    // Minknow will try to revert into the READY state to try to stabalise itself, but there is no guarantee of
     // that succeeding
     ERROR_STATUS = 0;
 
@@ -388,16 +388,41 @@ enum AcquisitionStopReason {
     // The device or asic was disconnected, causing the run to stop.
     STOPPED_DEVICE_ERROR = 6;
 
-    // The temperature was out of a safe range (either got extremely hot or was too 
+    // The temperature was out of a safe range (either got extremely hot or was too
     // far from the set-point for a certain amount of time)
     STOPPED_BAD_TEMPERATURE = 7;
+
+    // MinKNOW (or the sequencing device) was shut down, turned off or crashed.
+    STOPPED_SHUTDOWN = 8;
+}
+
+enum StartupState {
+    STARTUP_UNKNOWN = 0;
+
+    // Building the analysis pipeline objects.
+    STARTUP_BUILDING_PIPELINE = 2;
+
+    // Waiting for the basecaller to load any required data/references.
+    STARTUP_INITIALISING_BASECALLER = 3;
+
+    // Waiting for the basecaller to load alignment specific data files.
+    STARTUP_INITIALISING_BASECALLER_ALIGNMENT = 4;
+
+    // Setting up data writers for outputting reads/files.
+    STARTUP_INITIALISING_DATA_WRITERS = 5;
+
+    // Initialsing storage for in flight reads.
+    STARTUP_INITIALISING_INTERMEDIATE_DATA_STORAGE = 6;
+
+    // Initialsing storage for run statistics.
+    STARTUP_INITIALISING_STATISTICS = 7;
 }
 
 enum FinishingState {
     // The default state for acquisition
     FINISHING_UNKNOWN = 0;
 
-    // The acquisition is processing raw signal into reads 
+    // The acquisition is processing raw signal into reads
     FINISHING_PROCESSING_DEVICE_SIGNAL = 1;
 
     // Processed reads are being basecalled
@@ -473,6 +498,15 @@ message AcquisitionYieldSummary
     //
     // Since 4.0
     int64 alignment_deletions = 17;
+
+    // Number of bases that match the target reference(s) expressed as a
+    // fraction of the total size of the target reference(s).
+    //
+    // eg: For a specified alignment-targets with 2000 and 3000 bases, if
+    // "alignment_matches" is 2500, then "alignment_coverage" will be 0.5
+    //
+    // Since 4.3
+    float alignment_coverage = 19;
 }
 
 message AcquisitionWriterSummary
@@ -549,14 +583,31 @@ message ChannelStateInfo {
 }
 
 message AcquisitionConfigSummary {
+    // The purpose, as supplied to `acquisition.start()`
+    Purpose purpose = 21;
+
     // Was basecalling enabled for the run.
     bool basecalling_enabled = 1;
+    // Basecalling configuration filename (if basecalling enabled)
+    string basecalling_config_filename = 16;
+
     // Is barcoding enabled for the run
     bool barcoding_enabled = 13;
+    /// Barcoding kit(s) used (if barcoding enabled)
+    repeated string barcoding_kits = 17;
+
     // Is alignment enabled for the run
     bool alignment_enabled = 14;
+    // Alignment reference file(s) used (if alignment enabled)
+    repeated string alignment_reference_files = 18;
+    // bed file used (if alignment enabled, and bed file specified)
+    string alignment_bed_file = 19;
+
+
     // Is lamp enabled for the run
     bool lamp_enabled = 15;
+    // The LAMP kit used (if LAMP enabled)
+    string lamp_kit = 20;
 
     // Root directory reads were written to for the run.
     //
@@ -607,6 +658,9 @@ message AcquisitionRunInfo {
     // acquisitions on all MinKNOW instances.
     string run_id = 1;
 
+    // Current startup task (or STARTUP_UNKNOWN if not starting up).
+    StartupState startup_state = 12;
+
     // Indicates the current state of the acquisition.
     AcquisitionState state = 2;
 
@@ -672,21 +726,21 @@ message SetSignalReaderRequest {
         HDF5 = 0; // Default value
         DEVICE = 1;
     }
-    
+
     // The type of signal reader to use
     SignalReaderType reader = 1 [(rpc_required) = true];
-    
+
     // The following settings are optional, and only used when setting the reader to hdf5
     string hdf_source = 2;
-    
+
     enum SourceFileMode {
         UNSPECIFIED = 0; // Default value
         SINGLE_RUN = 1;
         LOOP = 2;
     }
-    
+
     SourceFileMode hdf_mode = 3; // Defaults to UNSPECIFIED, since this setting is optional
-    
+
     float sample_rate_scale_factor = 4;
 }
 

proto/minknow_api/analysis_configuration.proto

@@ -322,6 +322,7 @@ message ReadClassificationParams {
 message ChannelStates {
     message Logic {
         message Behaviour {
+            // TODO: MinKNOW 5: replace int32 with bool for these options
             // these are ints but act like bools
             int32 reset_on_mux_change = 1;
             int32 reset_on_well_change = 2;
@@ -547,6 +548,11 @@ message BarcodingConfiguration
     //
     // Maximum value is 100, defaults to 60.
     google.protobuf.FloatValue min_score_mid = 7;
+
+    // The minimum score required for the barcode mask to be detected.
+    //
+    // Maximum value is 100, defaults to 40. 
+    google.protobuf.FloatValue min_score_mask = 8;
 }
 
 // Since 4.0
@@ -779,10 +785,25 @@ message WriterConfiguration {
             repeated int32 channels = 1;
         }
 
+        message ChannelRanges
+        {
+            message ChannelRange
+            {
+                int32 start = 1;
+                int32 end = 2;
+            }
+            // List of start/end paired channel numbers
+            // which should be enabled for writing.
+            //
+            // All channels in inclusive ranges should be enabled.
+            repeated ChannelRange ranges = 1;
+        }
+
         // Control the way channels are enabled for this data type.
         oneof channels {
             bool all_channels = 1;
             ChannelList specific_channels = 2;
+            ChannelRanges channel_ranges = 3;
         }
     }
 
@@ -991,7 +1012,7 @@ message WriterConfiguration {
         // The pattern used to find the pdf report filename. If left empty but output is
         // enabled a default pattern is used.
         //
-        // default: report_{flow_cell_id}_{start_time}_{short_protocol_run_id}.pdf
+        // default: report_{flow_cell_id}_{daq_start_time}_{short_protocol_run_id}.pdf
         // Where each {xxx} section is replaced with an attribute from the minknow
         // state when the file is written.
         //
@@ -1001,7 +1022,7 @@ message WriterConfiguration {
         // The pattern used to find the markdown report filename. If left empty but output is
         // enabled a default pattern is used.
         //
-        // default: report_{flow_cell_id}_{start_time}_{short_protocol_run_id}.md
+        // default: report_{flow_cell_id}_{daq_start_time}_{short_protocol_run_id}.md
         // Where each {xxx} section is replaced with an attribute from the minknow
         // state when the file is written.
         //
@@ -1053,7 +1074,7 @@ message WriterConfiguration {
         // default: "_{flow_cell_id}_{short_run_id}"
         // Where each {xxx} section is replaced with an attribute from the minknow
         // state when the file is written.
-        // 
+        //
         // Custom reports use this to build filenames:
         //   - "custom_report{suffix}.txt"
         //

proto/minknow_api/basecaller.proto

@@ -66,6 +66,14 @@ service Basecaller {
     rpc watch (WatchRequest) returns (stream WatchResponse) {
         option idempotency_level = NO_SIDE_EFFECTS;
     }
+
+    // Build an alignment index file from an input fasta reference.
+    //
+    // This call blocks whilst the index is built.
+    //
+    // Since 4.3
+    rpc make_alignment_index(MakeAlignmentIndexRequest) returns (MakeAlignmentIndexResponse) {}
+
 }
 
 message ListConfigsByKitRequest {
@@ -88,6 +96,9 @@ message ListConfigsByKitResponse {
 }
 
 message StartBasecallingRequest {
+    // User specified name to identify the basecall run.
+    string name = 12;
+
     // Input directories to search for reads to be basecalled.
     //
     // Currently, only one directory can be specified, but this definition allows for multiple in
@@ -139,6 +150,9 @@ message StartBasecallingResponse {
 }
 
 message StartBarcodingRequest {
+    // User specified name to identify the barcoding run.
+    string name = 11;
+
     // Input directories to search for reads to be basecalled.
     //
     // Currently, only one directory can be specified, but this definition allows for multiple in
@@ -171,6 +185,9 @@ message StartBarcodingResponse {
 }
 
 message StartAlignmentRequest {
+    // User specified name to identify the alignment run.
+    string name = 7;
+
     // Input directories to search for reads to be aligned.
     //
     // Currently, only one directory can be specified, but this definition allows for multiple in
@@ -326,3 +343,14 @@ message WatchResponse {
     repeated RunInfo runs = 1;
 }
 
+message MakeAlignmentIndexRequest {
+    // Input fasta reference to use for building the index.
+    string input_alignment_reference = 1;
+
+    // Output file path to write index (mmi file) to.
+    //
+    // Must have a ".mmi" extension, and the paths parent directory must exist.
+    string output_alignment_index = 2;
+}
+
+message MakeAlignmentIndexResponse { }

proto/minknow_api/data.proto

@@ -878,6 +878,7 @@ message GetExperimentYieldInfoResponse{
         uint64 processed_skipped_reads = 8;
         uint64 processed_force_skipped_reads = 9;
         uint64 stored_reads_bytes_memory = 10;
+        uint64 stored_read_supporting_bytes_memory = 16;
         uint64 stored_reads_bytes_disk = 11;
         uint64 discarded_error_bytes = 12;
         uint64 channels_writing_to_disk = 13;
@@ -909,6 +910,7 @@ message GetExperimentYieldInfoResponse{
         uint64 pending_writes = 2;
         uint64 pending_hdf_tasks = 3;
         uint64 completed_writes = 4;
+        uint64 dataset_bytes_data_in_flight = 5;
     }
 
     message BasecallStatistics {