Merge pull request #6 from arabidopsis/main

Syncing new web code

Makefile

@@ -1,20 +1,20 @@
 # run this if there is a DEALER/ROUTER frontend running
 # see run-broker or run-chloe-broker:
 run-chloe:
-	JULIA_NUM_THREADS=8 julia --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --broker=@ipc:///tmp/chloe-client
+	julia --threads=8 --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --broker=@ipc:///tmp/chloe-client
 
 # start up chloe with a broker in the background
 run-chloe-broker:
-	JULIA_NUM_THREADS=8 julia --project=. --color=yes distributed.jl -l info --workers=4 --broker=ipc:///tmp/chloe-client
+	julia --threads=8 --project=. --color=yes distributed.jl -l info --workers=4 --broker=ipc:///tmp/chloe-client
 
 # just run the broker
 run-broker:
 	julia --project=. -q --startup-file=no src/broker.jl --worker=tcp://127.0.0.1:9467 --client=ipc:///tmp/chloe-client
 
 run-chloe-logger:
-	JULIA_NUM_THREADS=8 julia --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --broker=ipc:///tmp/chloe-client --backend=ipc:///tmp/chloe-logger
+	julia --threads=8 --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --broker=ipc:///tmp/chloe-client --backend=ipc:///tmp/chloe-logger
 
 run-chloe-backend:
-	JULIA_NUM_THREADS=8 julia --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --backend=ipc:///tmp/chloe-backend --broker=@ipc:///tmp/chloe-client
+	julia --threads=8 --project=. --color=yes distributed.jl -l info --workers=4 --address=tcp://127.0.0.1:9467 --backend=ipc:///tmp/chloe-backend --broker=@ipc:///tmp/chloe-client
 
 .PHONY: run-chloe run-chloe-broker run-broker run-chloe-logger run-chloe-backend 

bin/regression.sh

@@ -10,7 +10,7 @@ G='\e[1;32m'
 R='\e[1;31m'
 
 echo "start annotations..."
-JULIA_NUM_THREADS=8 time -p julia --project=. "$@" chloe.jl -l info annotate -o testo testfa/*.fa
+time -p julia --project=. --threads=8 "$@" chloe.jl -l info annotate -o testo --numgsrefs 16 testfa/*.fa
 for f in $(ls testo)
 do 
     echo "diffing $f"

bin/rregression.sh

@@ -49,7 +49,7 @@ done
 echo -e "index: ${index[@]}: $TOTAL/${C}$n${O}"
 echo "start annotations with: ${todo[@]}"
 
-JULIA_NUM_THREADS=8 time -p julia --project=. "$@" chloe.jl -l warn annotate -o testo "${todo[@]}"
+time -p julia --threads=8 --project=. "$@" chloe.jl -l warn annotate -o testo --numgsrefs 16 "${todo[@]}"
 
 for idx in ${index[@]}
 do

bin/tregression.sh

@@ -11,7 +11,7 @@ C='\e[1;36m' # bold cyan
 A='\e[1;30m' # grey
 
 echo -e "testing ${C}${#}${O} files"
-JULIA_NUM_THREADS=8 time -p julia --project=. chloe.jl -l warn annotate -o testo "$@"
+time -p julia --threads=8 --project=. chloe.jl -l warn annotate -o testo --numgsrefs 16 "$@"
 
 for f in "$@"
 do

src/Chloe.jl

@@ -19,8 +19,8 @@ include("dist/tasks.jl")
 include("dist/chloe_distributed.jl")
 
 # import .ChloeDistributed: distributed_main, chloe_distributed, run_broker, get_distributed_args, maybe_launch_broker
-import .Annotator: annotate, annotate_one, MayBeIO, MayBeString, Feature, ReferenceDb
-import .CmdLine: cmd_main
-import .ZMQLogging: set_global_logger
+import .Annotator: annotate, annotate_one, MayBeIO, MayBeString, Feature, ReferenceDb, AbstractReferenceDb
+import .CmdLine:cmd_main
+import .ZMQLogging:set_global_logger
 import .Annotator: read_single_reference!, inverted_repeat
 end

src/annotate_genomes.jl

@@ -2,7 +2,8 @@ module Annotator
 
 using Base: String
 using XGBoost
-export annotate, annotate_one, MayBeIO, MayBeString, ReferenceDb
+export annotate, annotate_one, MayBeIO, MayBeString, AbstractReferenceDb
+
 export read_single_reference!, inverted_repeat, ChloeConfig
 
 import Base
@@ -323,7 +324,7 @@ struct ChloeAnnotation
     coverages::Dict{String,Float32}
     annotation::FwdRev{Vector{SFF_Model}}
 end
-function annotate_one_worker(db::ReferenceDb,
+function annotate_one_worker(db::AbstractReferenceDb,
     target_id::String,
     target::FwdRev{CircularSequence},
     config::ChloeConfig,
@@ -345,12 +346,7 @@ function annotate_one_worker(db::ReferenceDb,
         append!(refpicks, searchhashes(hash, refhashes)[1:numrefs])
     end
 
-    # find closest chloe references
-    refhashes = get_chloeminhashes(db, config)
-    if !isnothing(refhashes)
-        numrefs = min(config.numchloerefs, length(refhashes))
-        append!(refpicks, searchhashes(hash, refhashes)[1:numrefs])
-    end
+
     numrefs = length(refpicks)
     t2 = time_ns()
 
@@ -359,7 +355,7 @@ function annotate_one_worker(db::ReferenceDb,
     blocks_aligned_to_targetf = Vector{FwdRev{BlockTree}}(undef, numrefs)
     blocks_aligned_to_targetr = Vector{FwdRev{BlockTree}}(undef, numrefs)
     coverages = Dict{String,Float32}()
-    refs = Vector{SingleReference}(undef, 0)
+    # refs = Vector{SingleReference}(undef, 0)
     reference_feature_counts = Dict{String,Int}()
     # get_single_reference! throws if bad refpick
     refs = [get_single_reference!(db, r[1], reference_feature_counts) for r in refpicks]
@@ -469,7 +465,7 @@ returns a 2-tuple: (ultimate sff output filename, sequence id)
 If `output_sff_file` is an IOBuffer then that buffer will be returned
 with the annotation within it.
 """
-function annotate_one(db::ReferenceDb,
+function annotate_one(db::AbstractReferenceDb,
     target_id::String,
     target::FwdRev{CircularSequence},
     config::ChloeConfig,
@@ -483,7 +479,7 @@ end
 
 
 
-function annotate_one(db::ReferenceDb, infile::String, config::ChloeConfig, output::MayBeIO=nothing)
+function annotate_one(db::AbstractReferenceDb, infile::String, config::ChloeConfig, output::MayBeIO=nothing)
     maybe_gzread(infile) do io
         annotate_one(db, io, config, output)
     end
@@ -513,7 +509,7 @@ function fasta_reader(infile::IO)::Tuple{String,FwdRev{CircularSequence}}
     return target_id, FwdRev(fseq, rseq)
 end
 
-function annotate_one(db::ReferenceDb, infile::IO, config::ChloeConfig, output::MayBeIO=nothing)
+function annotate_one(db::AbstractReferenceDb, infile::IO, config::ChloeConfig, output::MayBeIO=nothing)
     target_id, seqs = fasta_reader(infile)
     annotate_one(db, target_id, seqs, config, output)
 end
@@ -533,11 +529,11 @@ end
 
 #= function annotate_one(refsdir::String, infile::String, output::MayBeIO=nothing)
 
-    annotate_one(ReferenceDb(;refsdir=refsdir), infile, ChloeConfig(), output)
+    annotate_one(AbstractReferenceDb(;refsdir=refsdir), infile, ChloeConfig(), output)
 
 end =#
 
-function annotate(db::ReferenceDb, fa_files::Vector{String}, config::ChloeConfig, output::Union{Nothing,String}=nothing)
+function annotate(db::AbstractReferenceDb, fa_files::Vector{String}, config::ChloeConfig, output::Union{Nothing,String}=nothing)
     n = length(fa_files)
     for infile in fa_files
         maybe_gzread(infile) do io

src/dist/ZMQLogger.jl

@@ -30,7 +30,7 @@ struct ZMQLogger <: Logging.AbstractLogger
     message_limits::Dict{Any,Int}
     socket::ZMQ.Socket
 
-    function ZMQLogger(endpoint::String, min_level::Logging.LogLevel = Logging.Warn; topic::String = "", message_limits = nothing)
+    function ZMQLogger(endpoint::String, min_level::Logging.LogLevel=Logging.Warn; topic::String="", message_limits=nothing)
         if message_limits === nothing
             message_limits = Dict{Any,Int}()
         end
@@ -42,7 +42,7 @@ struct ZMQLogger <: Logging.AbstractLogger
 end
 
 function Logging.handle_message(logger::ZMQLogger, level, message, _module, group, id,
-    filepath, line; maxlog = nothing, kwargs...)
+    filepath, line; maxlog=nothing, kwargs...)
 
     if maxlog !== nothing && maxlog isa Integer
         remaining = get!(logger.message_limits, id, maxlog)
@@ -53,8 +53,8 @@ function Logging.handle_message(logger::ZMQLogger, level, message, _module, grou
     buf = IOBuffer()
     io = IOContext(buf, stderr)
     println(io, msglines[1])
-    for i in 2:length(msglines)
-        println(io, "\t> ", msglines[i])
+    for v in msglines[2:end]
+        println(io, "\t> ", v)
     end
     for (key, val) in kwargs
         println(io, "\t> ", key, " = ", val)
@@ -70,8 +70,8 @@ function Logging.handle_message(logger::ZMQLogger, level, message, _module, grou
             topic = "$(topic).$(task_id)"
         end
         # julia Strings are already utf-8: Vector{UInt8}(msg)
-        ZMQ.send(logger.socket, ZMQ.Message(topic); more = true)
-        ZMQ.send(logger.socket, ZMQ.Message(msg); more = false)
+        ZMQ.send(logger.socket, ZMQ.Message(topic); more=true)
+        ZMQ.send(logger.socket, ZMQ.Message(msg); more=false)
     end
     nothing
 end
@@ -88,7 +88,7 @@ function Logging.catch_exceptions(logger::ZMQLogger)
 end
 
 
-function set_global_logger(level::String = "warn", endpoint::MayBeString = nothing; quiet::Bool = true, topic = "")
+function set_global_logger(level::String="warn", endpoint::MayBeString=nothing; quiet::Bool=true, topic="")
 
     # don't add any line number guff even for debugging
     function quiet_metafmt(level, _module, group, id, file, line)
@@ -100,10 +100,10 @@ function set_global_logger(level::String = "warn", endpoint::MayBeString = nothi
     llevel = get(LOGLEVELS, level, Logging.Warn)
 
     if endpoint === nothing
-        logger = Logging.ConsoleLogger(stderr, llevel, meta_formatter = quiet ? quiet_metafmt : Logging.default_metafmt)
+        logger = Logging.ConsoleLogger(stderr, llevel, meta_formatter=quiet ? quiet_metafmt : Logging.default_metafmt)
     else
-        logger = ZMQLogger(endpoint::String, llevel; topic = topic)
+        logger = ZMQLogger(endpoint::String, llevel; topic=topic)
     end
     Logging.global_logger(logger)
 end
-end # module
+end # module

src/dist/broker.jl

@@ -60,7 +60,7 @@ function start_broker(worker_url::String, client_url::String)
     try
         try
             @info "worker=$worker_url client=$client_url"
-            rc = ccall((:zmq_proxy, ZeroMQ_jll.libzmq), Cint,  (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}), router, dealer, C_NULL)
+            rc = ccall((:zmq_proxy, ZeroMQ_jll.libzmq), Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}), router, dealer, C_NULL)
             @info "done zmq_proxy $(rc)"
         catch
             @error "zmq_proxy exception!"

src/dist/chloe_cmd.jl

@@ -16,16 +16,16 @@ function quiet_metafmt(level, _module, group, id, file, line)
     return color, prefix, ""
 end
 
-function chloe(; gsrefsdir = "default", chloerefsdir = "default", numgsrefs = DEFAULT_NUMGSREFS, numchloerefs = DEFAULT_NUMGSREFS, fasta_files = String[],
-    template = "default", sensitivity = DEFAULT_SENSITIVITY,
-    output::Union{Nothing,String} = nothing, gff::Bool = false, nofilter::Bool = false)
-    db = Annotator.ReferenceDb(; gsrefsdir = gsrefsdir, chloerefsdir = chloerefsdir, template = template)
-    config = Annotator.ChloeConfig(; numgsrefs = numgsrefs, numchloerefs = numchloerefs, sensitivity = sensitivity, to_gff3 = gff, nofilter = nofilter)
+function chloe(; gsrefsdir="default", numgsrefs=DEFAULT_NUMGSREFS, fasta_files=String[],
+    template="default", sensitivity=DEFAULT_SENSITIVITY,
+    output::Union{Nothing,String}=nothing, gff::Bool=false, nofilter::Bool=false)
+    db = Annotator.ReferenceDb(; gsrefsdir=gsrefsdir, template=template)
+    config = Annotator.ChloeConfig(; numgsrefs=numgsrefs, sensitivity=sensitivity, to_gff3=gff, nofilter=nofilter)
     Annotator.annotate(db, fasta_files, config, output)
 end
 
 function getargs()
-    cmd_args = ArgParseSettings(prog = "Chloë", autofix_names = true)  # turn "-" into "_" for arg names.
+    cmd_args = ArgParseSettings(prog="Chloë", autofix_names=true)  # turn "-" into "_" for arg names.
 
     @add_arg_table! cmd_args begin
         "minhash"
@@ -86,19 +86,14 @@ function getargs()
         "--reference", "-r"
         arg_type = String
         default = "default"
-        dest_name = "chloerefsdir"
+        dest_name = "gsrefsdir"
         metavar = "DIRECTORY"
-        help = "reference directory [default: $(DEFAULT_CHLOEREFS)]"
+        help = "reference directory [default: $(DEFAULT_GSREFS)]"
         "--numgsrefs"
         arg_type = Int
         default = DEFAULT_NUMGSREFS
         dest_name = "numgsrefs"
         help = "number of references to compare to [default: $(DEFAULT_NUMGSREFS)]"
-        "--numchloerefs"
-        arg_type = Int
-        default = DEFAULT_NUMCHLOEREFS
-        dest_name = "numchloerefs"
-        help = "number of references to compare to [default: $(DEFAULT_NUMCHLOEREFS)]"
         "--template", "-t"
         arg_type = String
         default = "default"
@@ -143,14 +138,14 @@ function getargs()
     #     examples:\n
     #     \ua0\ua0 # chloe.jl -t template.tsv -r reference_dir fasta1 fasta2 ...\n
     #     """
-    parse_args(ARGS, cmd_args; as_symbols = true)
+    parse_args(ARGS, cmd_args; as_symbols=true)
 end
 
 function cmd_main()
     parsed_args = getargs()
     level = lowercase(parsed_args[:level])
     Logging.with_logger(Logging.ConsoleLogger(stderr,
-        get(LOGLEVELS, level, Logging.Warn), meta_formatter = quiet_metafmt)) do
+        get(LOGLEVELS, level, Logging.Warn), meta_formatter=quiet_metafmt)) do
         cmd = parsed_args[:_COMMAND_]
         a = parsed_args[cmd]
         if cmd == :minhash