diff --git a/.github/workflows/qleverfiles-check.yml b/.github/workflows/qleverfiles-check.yml index 528beb14..4cda5f8d 100644 --- a/.github/workflows/qleverfiles-check.yml +++ b/.github/workflows/qleverfiles-check.yml @@ -33,6 +33,7 @@ jobs: - name: Check that all the files in `src/qlever/Qleverfiles` parse. working-directory: ${{github.workspace}}/qlever-control run: | + export QLEVER_ARGCOMPLETE_ENABLED=1 for QLEVERFILE in src/qlever/Qleverfiles/Qleverfile.*; do echo echo -e "\x1b[1;34mChecking ${QLEVERFILE}\x1b[0m" diff --git a/src/qlever/Qleverfiles/Qleverfile.dblp b/src/qlever/Qleverfiles/Qleverfile.dblp index d7e90937..574d1192 100644 --- a/src/qlever/Qleverfiles/Qleverfile.dblp +++ b/src/qlever/Qleverfiles/Qleverfile.dblp @@ -17,7 +17,7 @@ FORMAT = ttl [index] INPUT_FILES = *.gz -MULTI_INPUT_JSON = $$(ls *.gz | xargs -I {} echo '{ "cmd": "zcat {}" }') +MULTI_INPUT_JSON = { "cmd": "zcat {}", "for-each": "*.gz" } SETTINGS_JSON = { "ascii-prefixes-only": false, "num-triples-per-batch": 5000000, "prefixes-external": [""] } [server] diff --git a/src/qlever/commands/add_text_index.py b/src/qlever/commands/add_text_index.py index 15fa647b..f2401878 100644 --- a/src/qlever/commands/add_text_index.py +++ b/src/qlever/commands/add_text_index.py @@ -64,7 +64,7 @@ def execute(self, args) -> bool: # Show the command line. self.show(add_text_index_cmd, only_show=args.show) if args.show: - return False + return True # When running natively, check if the binary exists and works. if args.system == "native": @@ -74,6 +74,7 @@ def execute(self, args) -> bool: log.error(f"Running \"{args.index_binary}\" failed ({e}), " f"set `--index-binary` to a different binary or " f"use `--container_system`") + return False # Check if text index files already exist. existing_text_index_files = get_existing_index_files( diff --git a/src/qlever/commands/cache_stats.py b/src/qlever/commands/cache_stats.py index 28eb6e58..5b618a98 100644 --- a/src/qlever/commands/cache_stats.py +++ b/src/qlever/commands/cache_stats.py @@ -47,7 +47,7 @@ def execute(self, args) -> bool: self.show("\n".join([cache_stats_cmd, cache_settings_cmd]), only_show=args.show) if args.show: - return False + return True # Execute them. try: diff --git a/src/qlever/commands/clear_cache.py b/src/qlever/commands/clear_cache.py index 448aeb39..8732120c 100644 --- a/src/qlever/commands/clear_cache.py +++ b/src/qlever/commands/clear_cache.py @@ -48,7 +48,7 @@ def execute(self, args) -> bool: f"\"{args.access_token}\"") self.show(clear_cache_cmd, only_show=args.show) if args.show: - return False + return True # Execute the command. try: @@ -76,5 +76,7 @@ def execute(self, args) -> bool: # Show cache stats. log.info("") args.detailed = False - CacheStatsCommand().execute(args) + if not CacheStatsCommand().execute(args): + log.error("Clearing the cache was successful, but showing the " + "cache stats failed {e}") return True diff --git a/src/qlever/commands/example_queries.py b/src/qlever/commands/example_queries.py index c38b8678..dff44ba0 100644 --- a/src/qlever/commands/example_queries.py +++ b/src/qlever/commands/example_queries.py @@ -197,7 +197,7 @@ def execute(self, args) -> bool: only_show=args.show, ) if args.show: - return False + return True # Get the example queries. try: @@ -229,8 +229,11 @@ def execute(self, args) -> bool: if args.clear_cache == "yes": args.server_url = sparql_endpoint args.complete = False + clear_cache_successful = False with mute_log(): - ClearCacheCommand().execute(args) + clear_cache_successful = ClearCacheCommand().execute(args) + if not clear_cache_successful: + log.warn("Failed to clear the cache") # Remove OFFSET and LIMIT (after the last closing bracket). if args.remove_offset_and_limit or args.limit: diff --git a/src/qlever/commands/get_data.py b/src/qlever/commands/get_data.py index 4ae2bb7d..b27eca5f 100644 --- a/src/qlever/commands/get_data.py +++ b/src/qlever/commands/get_data.py @@ -31,7 +31,7 @@ def execute(self, args) -> bool: # Construct the command line and show it. self.show(args.get_data_cmd, only_show=args.show) if args.show: - return False + return True # Execute the command line. try: diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 81babea1..2ca8aaff 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -239,7 +239,7 @@ def execute(self, args) -> bool: # Show the command line. self.show(f"{settings_json_cmd}\n{index_cmd}", only_show=args.show) if args.show: - return False + return True # When running natively, check if the binary exists and works. if args.system == "native": diff --git a/src/qlever/commands/index_stats.py b/src/qlever/commands/index_stats.py index 975576ac..b997b8c7 100644 --- a/src/qlever/commands/index_stats.py +++ b/src/qlever/commands/index_stats.py @@ -18,32 +18,45 @@ def __init__(self): pass def description(self) -> str: - return ("Breakdown of the time and space used for the index build") + return "Breakdown of the time and space used for the index build" def should_have_qleverfile(self) -> bool: return False - def relevant_qleverfile_arguments(self) -> dict[str: list[str]]: + def relevant_qleverfile_arguments(self) -> dict[str : list[str]]: return {"data": ["name"]} def additional_arguments(self, subparser) -> None: - subparser.add_argument("--only-time", action="store_true", - default=False, - help="Show only the time used") - subparser.add_argument("--only-space", action="store_true", - default=False, - help="Show only the space used") - subparser.add_argument("--ignore-text-index", action="store_true", - default=False, - help="Ignore the text index") - subparser.add_argument("--time-unit", - choices=["s", "min", "h", "auto"], - default="auto", - help="The time unit") - subparser.add_argument("--size-unit", - choices=["B", "MB", "GB", "TB", "auto"], - default="auto", - help="The size unit") + subparser.add_argument( + "--only-time", + action="store_true", + default=False, + help="Show only the time used", + ) + subparser.add_argument( + "--only-space", + action="store_true", + default=False, + help="Show only the space used", + ) + subparser.add_argument( + "--ignore-text-index", + action="store_true", + default=False, + help="Ignore the text index", + ) + subparser.add_argument( + "--time-unit", + choices=["s", "min", "h", "auto"], + default="auto", + help="The time unit", + ) + subparser.add_argument( + "--size-unit", + choices=["B", "MB", "GB", "TB", "auto"], + default="auto", + help="The size unit", + ) def execute_time(self, args, log_file_name) -> bool: """ @@ -65,8 +78,9 @@ def execute_time(self, args, log_file_name) -> bool: with open(text_log_file_name, "r") as text_log_file: lines.extend(text_log_file.readlines()) except Exception as e: - log.error(f"Problem reading text index log file " - f"{text_log_file_name}: {e}") + log.error( + f"Problem reading text index log file " f"{text_log_file_name}: {e}" + ) return False # Helper function that finds the next line matching the given `regex`, @@ -95,12 +109,14 @@ def find_next_line(regex, update_current_line=True): if regex_match: try: return datetime.strptime( - re.match(timestamp_regex, line).group(), - timestamp_format), regex_match + re.match(timestamp_regex, line).group(), timestamp_format + ), regex_match except Exception as e: - log.error(f"Could not parse timestamp of form " - f"\"{timestamp_regex}\" from line " - f" \"{line.rstrip()}\" ({e})") + log.error( + f"Could not parse timestamp of form " + f'"{timestamp_regex}" from line ' + f' "{line.rstrip()}" ({e})' + ) # If we get here, we did not find a matching line. if not update_current_line: current_line = current_line_backup @@ -119,26 +135,32 @@ def find_next_line(regex, update_current_line=True): # file (old format: "Creating a pair" + names of permutations in # line "Writing meta data for ..."; new format: name of # permutations already in line "Creating permutations ..."). - perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair", - update_current_line=False) + perm_begin, _ = find_next_line( + r"INFO:\s*Creating a pair", update_current_line=False + ) if perm_begin is None: perm_begin, perm_info = find_next_line( r"INFO:\s*Creating permutations ([A-Z]+ and [A-Z]+)", - update_current_line=False) + update_current_line=False, + ) else: _, perm_info = find_next_line( r"INFO:\s*Writing meta data for ([A-Z]+ and [A-Z]+)", - update_current_line=False) + update_current_line=False, + ) if perm_info is None: break perm_begin_and_info.append((perm_begin, perm_info)) - convert_end = (perm_begin_and_info[0][0] if - len(perm_begin_and_info) > 0 else None) + convert_end = ( + perm_begin_and_info[0][0] if len(perm_begin_and_info) > 0 else None + ) normal_end, _ = find_next_line(r"INFO:\s*Index build completed") - text_begin, _ = find_next_line(r"INFO:\s*Adding text index", - update_current_line=False) - text_end, _ = find_next_line(r"INFO:\s*Text index build comp", - update_current_line=False) + text_begin, _ = find_next_line( + r"INFO:\s*Adding text index", update_current_line=False + ) + text_end, _ = find_next_line( + r"INFO:\s*Text index build comp", update_current_line=False + ) if args.ignore_text_index: text_begin = text_end = None @@ -147,9 +169,11 @@ def find_next_line(regex, update_current_line=True): log.error("Missing line that index build has started") return False if overall_begin and not merge_begin: - log.error("According to the log file, the index build " - "has started, but is still in its first " - "phase (parsing the input)") + log.error( + "According to the log file, the index build " + "has started, but is still in its first " + "phase (parsing the input)" + ) return False # Helper function that shows the duration for a phase (if the start and @@ -187,22 +211,24 @@ def show_duration(heading, start_end_pairs): show_duration("Convert to global IDs", [(convert_begin, convert_end)]) for i in range(len(perm_begin_and_info)): perm_begin, perm_info = perm_begin_and_info[i] - perm_end = perm_begin_and_info[i + 1][0] if i + 1 < len( - perm_begin_and_info) else normal_end - perm_info_text = (perm_info.group(1).replace(" and ", " & ") - if perm_info else f"#{i + 1}") - show_duration(f"Permutation {perm_info_text}", - [(perm_begin, perm_end)]) + perm_end = ( + perm_begin_and_info[i + 1][0] + if i + 1 < len(perm_begin_and_info) + else normal_end + ) + perm_info_text = ( + perm_info.group(1).replace(" and ", " & ") if perm_info else f"#{i + 1}" + ) + show_duration(f"Permutation {perm_info_text}", [(perm_begin, perm_end)]) show_duration("Text index", [(text_begin, text_end)]) if text_begin and text_end: log.info("") - show_duration("TOTAL time", - [(overall_begin, normal_end), - (text_begin, text_end)]) + show_duration( + "TOTAL time", [(overall_begin, normal_end), (text_begin, text_end)] + ) elif normal_end: log.info("") - show_duration("TOTAL time", - [(overall_begin, normal_end)]) + show_duration("TOTAL time", [(overall_begin, normal_end)]) return True def execute_space(self, args) -> bool: @@ -252,24 +278,29 @@ def show_size(heading, size): return True def execute(self, args) -> bool: - ret_value = args.show + return_value = True # The "time" part of the command. if not args.only_space: log_file_name = f"{args.name}.index-log.txt" - self.show(f"Breakdown of the time used for " - f"building the index, based on the timestamps for key " - f"lines in \"{log_file_name}\"", only_show=args.show) + self.show( + f"Breakdown of the time used for " + f"building the index, based on the timestamps for key " + f'lines in "{log_file_name}"', + only_show=args.show, + ) if not args.show: - ret_value &= self.execute_time(args, log_file_name) + return_value &= self.execute_time(args, log_file_name) if not args.only_time: log.info("") # The "space" part of the command. if not args.only_time: - self.show("Breakdown of the space used for building the index", - only_show=args.show) + self.show( + "Breakdown of the space used for building the index", + only_show=args.show, + ) if not args.show: - ret_value &= self.execute_space(args) + return_value &= self.execute_space(args) - return ret_value + return return_value diff --git a/src/qlever/commands/log.py b/src/qlever/commands/log.py index 3b2599b4..816072bc 100644 --- a/src/qlever/commands/log.py +++ b/src/qlever/commands/log.py @@ -47,10 +47,20 @@ def execute(self, args) -> bool: log_cmd += f" {log_file}" self.show(log_cmd, only_show=args.show) if args.show: - return False + return True # Execute the command. log.info(f"Follow log file {log_file}, press Ctrl-C to stop" f" following (will not stop the server)") log.info("") - subprocess.run(log_cmd, shell=True) + try: + subprocess.run(log_cmd, shell=True) + return True + except Exception as e: + log.error(e) + return False + + + + + diff --git a/src/qlever/commands/query.py b/src/qlever/commands/query.py index 163c0c8a..979cd5a5 100644 --- a/src/qlever/commands/query.py +++ b/src/qlever/commands/query.py @@ -53,7 +53,7 @@ def execute(self, args) -> bool: f" --data-urlencode query={shlex.quote(args.query)}") self.show(curl_cmd, only_show=args.show) if args.show: - return False + return True # Launch query. try: diff --git a/src/qlever/commands/setup_config.py b/src/qlever/commands/setup_config.py index 6cff7822..0eff3b23 100644 --- a/src/qlever/commands/setup_config.py +++ b/src/qlever/commands/setup_config.py @@ -60,7 +60,7 @@ def execute(self, args) -> bool: setup_config_cmd += "> Qleverfile" self.show(setup_config_cmd, only_show=args.show) if args.show: - return False + return True # If there is already a Qleverfile in the current directory, exit. qleverfile_path = Path("Qleverfile") diff --git a/src/qlever/commands/start.py b/src/qlever/commands/start.py index e51717b5..4b6ae8f6 100644 --- a/src/qlever/commands/start.py +++ b/src/qlever/commands/start.py @@ -72,7 +72,9 @@ def execute(self, args) -> bool: if args.kill_existing_with_same_port: args.cmdline_regex = f"^ServerMain.* -p {args.port}" args.no_containers = True - StopCommand().execute(args) + if not StopCommand().execute(args): + log.error("Stopping the existing server failed") + return False log.info("") # Construct the command line based on the config file. @@ -115,7 +117,7 @@ def execute(self, args) -> bool: # Show the command line. self.show(start_cmd, only_show=args.show) if args.show: - return False + return True # When running natively, check if the binary exists and works. if args.system == "native": @@ -192,6 +194,7 @@ def execute(self, args) -> bool: run_command(curl_cmd) except Exception as e: log.error(f"Setting the index description failed ({e})") + return False if args.text_description: text_desc = args.text_description curl_cmd = (f"curl -Gs http://localhost:{port}/api" @@ -202,6 +205,7 @@ def execute(self, args) -> bool: run_command(curl_cmd) except Exception as e: log.error(f"Setting the text description failed ({e})") + return False # Kill the tail process. NOTE: `tail_proc.kill()` does not work. tail_proc.terminate() @@ -209,7 +213,9 @@ def execute(self, args) -> bool: # Execute the warmup command. if args.warmup_cmd and not args.no_warmup: log.info("") - WarmupCommand().execute(args) + if not WarmupCommand().execute(args): + log.error("Warmup failed") + return False # Show cache stats. log.info("") diff --git a/src/qlever/commands/status.py b/src/qlever/commands/status.py index 5d066660..a8efed54 100644 --- a/src/qlever/commands/status.py +++ b/src/qlever/commands/status.py @@ -35,7 +35,7 @@ def execute(self, args) -> bool: f"the command line matches {args.cmdline_regex}" f" using Python's psutil library", only_show=args.show) if args.show: - return False + return True # Show the results as a table. num_processes_found = 0 @@ -47,3 +47,4 @@ def execute(self, args) -> bool: num_processes_found += 1 if num_processes_found == 0: print("No processes found") + return True diff --git a/src/qlever/commands/stop.py b/src/qlever/commands/stop.py index f2f8e80a..82225304 100644 --- a/src/qlever/commands/stop.py +++ b/src/qlever/commands/stop.py @@ -49,7 +49,7 @@ def execute(self, args) -> bool: f"\"{args.server_container}\"") self.show(description, only_show=args.show) if args.show: - return False + return True # First check if there is container running and if yes, stop and remove # it (unless the user has specified `--no-containers`). @@ -90,14 +90,12 @@ def execute(self, args) -> bool: return False return True - # No matching process found. + # If no matching process found, show a message and the output of the + # status command. message = "No matching process found" if args.no_containers else \ "No matching process or container found" log.error(message) - - # Show output of status command. args.cmdline_regex = "^ServerMain.* -i [^ ]*" log.info("") StatusCommand().execute(args) - - return False + return True diff --git a/src/qlever/commands/system_info.py b/src/qlever/commands/system_info.py index 0d1ed167..1e45d1bd 100644 --- a/src/qlever/commands/system_info.py +++ b/src/qlever/commands/system_info.py @@ -58,7 +58,7 @@ def execute(self, args) -> bool: # Say what the command is doing. self.show("Show system information and Qleverfile", only_show=args.show) if args.show: - return False + return True # Show system information. show_heading("System Information") diff --git a/src/qlever/commands/ui.py b/src/qlever/commands/ui.py index b4baa248..23dea87a 100644 --- a/src/qlever/commands/ui.py +++ b/src/qlever/commands/ui.py @@ -70,8 +70,10 @@ def execute(self, args) -> bool: "\n".join(["Stop running containers", pull_cmd, run_cmd, exec_cmd]), only_show=args.show, ) - if qlever_is_running_in_container or args.show: + if qlever_is_running_in_container: return False + if args.show: + return True # Stop running containers. for container_system in Containerize.supported_systems(): diff --git a/src/qlever/commands/warmup.py b/src/qlever/commands/warmup.py index 7a7041fb..49150262 100644 --- a/src/qlever/commands/warmup.py +++ b/src/qlever/commands/warmup.py @@ -30,7 +30,7 @@ def execute(self, args) -> bool: # Show what the command is doing. self.show(args.warmup_cmd, only_show=args.show) if args.show: - return False + return True # Execute the command. try: