[HWORKS-901] Git installation support should not prepend git+ (#1449)

logicalclocks · Jan 10, 2024 · fb4f442 · fb4f442
1 parent a6497a2
commit fb4f442
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 9 deletions.
diff --git a/hopsworks-IT/src/test/ruby/spec/conda_spec.rb b/hopsworks-IT/src/test/ruby/spec/conda_spec.rb
@@ -178,14 +178,30 @@
             delete_env(@project[:id], ENV['PYTHON_VERSION'])
             wait_for_running_command(@project[:id])
           end
-          it 'install from git' do
+          it 'install from git branch using @' do
+            expect(CondaCommands.find_by(project_id: @project[:id])).to be_nil
+            @project = create_env_and_update_project(@project, ENV['PYTHON_VERSION'])
+            uninstall_library(@project[:id], ENV['PYTHON_VERSION'], 'matplotlib')
+
+            wait_for_running_command(@project[:id])
+
+            install_library(@project[:id], ENV['PYTHON_VERSION'], '[email protected]', 'GIT', 'git', 'matplotlib @ git+https://github.com/matplotlib/[email protected]')
+
+            wait_for_running_command(@project[:id])
+
+            list_libraries(@project[:id], ENV['PYTHON_VERSION'])
+
+            matplotlib_library = json_body[:items].detect { |library| library[:library] == "matplotlib" }
+            expect(matplotlib_library[:version]).to eq "3.7.2"
+          end
+          it 'install from git branch using git+' do
             expect(CondaCommands.find_by(project_id: @project[:id])).to be_nil
             @project = create_env_and_update_project(@project, ENV['PYTHON_VERSION'])
             uninstall_library(@project[:id], ENV['PYTHON_VERSION'], 'hops')
 
             wait_for_running_command(@project[:id])
 
-            install_library(@project[:id], ENV['PYTHON_VERSION'], '[email protected]', 'GIT', 'git', 'https://github.com/logicalclocks/[email protected]')
+            install_library(@project[:id], ENV['PYTHON_VERSION'], '[email protected]', 'GIT', 'git', 'git+https://github.com/logicalclocks/[email protected]')
 
             wait_for_running_command(@project[:id])
 
@@ -194,6 +210,22 @@
             hops_library = json_body[:items].detect { |library| library[:library] == "hops" }
             expect(hops_library[:version]).to eq "2.0.0.2"
           end
+          it 'install from git branch using https://' do
+            expect(CondaCommands.find_by(project_id: @project[:id])).to be_nil
+            @project = create_env_and_update_project(@project, ENV['PYTHON_VERSION'])
+            uninstall_library(@project[:id], ENV['PYTHON_VERSION'], 'hops')
+
+            wait_for_running_command(@project[:id])
+
+            install_library(@project[:id], ENV['PYTHON_VERSION'], '[email protected]', 'GIT', 'git', 'https://github.com/logicalclocks/[email protected]')
+
+            wait_for_running_command(@project[:id])
+
+            list_libraries(@project[:id], ENV['PYTHON_VERSION'])
+
+            hops_library = json_body[:items].detect { |library| library[:library] == "hops" }
+            expect(hops_library[:version]).to eq "2.1.0.1"
+          end
           it 'install from wheel' do
             expect(CondaCommands.find_by(project_id: @project[:id])).to be_nil
             @project = create_env_and_update_project(@project, ENV['PYTHON_VERSION'])

diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java
@@ -67,8 +67,6 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -352,10 +350,9 @@ private void validateBundledDependency(Users user, LibrarySpecification libraryS
   }
 
   private void validateGitURL(String url) {
-    try {
-      new URL((url));
-    } catch (MalformedURLException e) {
-      throw new IllegalArgumentException("The provided Git URL is not a valid URL");
+    //url.startsWith("https://") is kept for backwards compatibility reasons
+    if(!url.toLowerCase().contains("git+") && !url.toLowerCase().startsWith("https://")) {
+      throw new IllegalArgumentException("The provided git installation url is not valid as it does not contain git+");
     }
   }
 }
diff --git a/...ommon/src/main/java/io/hops/hopsworks/common/python/environment/DockerFileController.java b/...ommon/src/main/java/io/hops/hopsworks/common/python/environment/DockerFileController.java
@@ -232,7 +232,11 @@ public BuildImageDetails installLibrary(File baseDir,
                   + apiToken + "@" + repoUrl.getHost() + repoUrl.getPath() + "'");
               }
             } else {
-              writer.write(anaconda_project_dir + "/bin/pip install --upgrade 'git+" + cc.getArg() + "'");
+              if(cc.getArg().startsWith("https://")) { //This is for backwards compatibility with previous behaviour
+                writer.write(anaconda_project_dir + "/bin/pip install --upgrade 'git+" + cc.getArg() + "'");
+              } else {
+                writer.write(anaconda_project_dir + "/bin/pip install --upgrade '" + cc.getArg() + "'");
+              }
             }
             dockerBuildOpts.add(DOCKER_NO_CACHE_OPT);
             break;