Add separate docs tarball limits and support infinity (#143)

hexpm · Feb 29, 2024 · 65590ff · 65590ff
1 parent 8ba1693
commit 65590ff
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 54 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -15,7 +15,7 @@ jobs:
       matrix:
         pair:
           - erlang: master
-            rebar3: 3.18.0
+            rebar3: 3.22.1
 
           - erlang: 25
             rebar3: 3.18.0

diff --git a/src/hex_core.erl b/src/hex_core.erl
@@ -46,6 +46,18 @@
 %%
 %% * `repo_verify_origin' - If `true' will verify the repository signature origin,
 %%   requires protobuf messages as of hex_core v0.4.0 (default: `true').
+%%
+%% * `tarball_max_size' - Maximum size of package tarball, defaults to
+%%   `16_777_216' (16 MiB). Set to `infinity' to not enforce the limit.
+%%
+%% * `tarball_max_uncompressed_size' - Maximum size of uncompressed package tarball, defaults to
+%%   `67_108_864' (64 MiB). Set to `infinity' to not enforce the limit.
+%%
+%% * `docs_tarball_max_size' - Maximum size of docs tarball, defaults to
+%%   `16_777_216' (16 MiB). Set to `infinity' to not enforce the limit.
+%%
+%% * `docs_tarball_max_uncompressed_size' - Maximum size of uncompressed docs tarball, defaults to
+%%   `134_217_728' (128 MiB). Set to `infinity' to not enforce the limit.
 
 -module(hex_core).
 -export([default_config/0]).
@@ -81,8 +93,10 @@
     repo_organization => binary() | undefined,
     repo_verify => boolean(),
     repo_verify_origin => boolean(),
-    tarball_max_size => pos_integer(),
-    tarball_max_uncompressed_size => pos_integer()
+    tarball_max_size => pos_integer() | infinity,
+    tarball_max_uncompressed_size => pos_integer() | infinity,
+    docs_tarball_max_size => pos_integer() | infinity,
+    docs_tarball_max_uncompressed_size => pos_integer() | infinity
 }.
 
 -spec default_config() -> config().
@@ -104,5 +118,7 @@ default_config() ->
         repo_verify => true,
         repo_verify_origin => true,
         tarball_max_size => 8 * 1024 * 1024,
-        tarball_max_uncompressed_size => 64 * 1024 * 1024
+        tarball_max_uncompressed_size => 64 * 1024 * 1024,
+        docs_tarball_max_size => 16 * 1024 * 1024,
+        docs_tarball_max_uncompressed_size => 128 * 1024 * 1024
     }.
diff --git a/src/hex_tarball.erl b/src/hex_tarball.erl
@@ -58,6 +58,11 @@
     }}
     | {error, term()}.
 create(Metadata, Files, Config) ->
+    #{
+        tarball_max_size := TarballMaxSize,
+        tarball_max_uncompressed_size := TarballMaxUncompressedSize
+    } = Config,
+
     MetadataBinary = encode_metadata(Metadata),
     ContentsTarball = create_memory_tarball(Files),
     ContentsTarballCompressed = gzip(ContentsTarball),
@@ -73,20 +78,23 @@ create(Metadata, Files, Config) ->
         {"contents.tar.gz", ContentsTarballCompressed}
     ],
 
-    Tarball = create_memory_tarball(OuterFiles),
-    OuterChecksum = checksum(Tarball),
-
-    UncompressedSize = byte_size(ContentsTarball),
-
-    case {(byte_size(Tarball) > TarballMaxSize), (UncompressedSize > TarballMaxUncompressedSize)} of
-        {_, true} ->
-            {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}};
-        {true, _} ->
-            {error, {tarball, {too_big_compressed, TarballMaxSize}}};
-        {false, false} ->
-            {ok, #{
-                tarball => Tarball, outer_checksum => OuterChecksum, inner_checksum => InnerChecksum
-            }}
+    case valid_size(ContentsTarball, TarballMaxUncompressedSize) of
+        true ->
+            Tarball = create_memory_tarball(OuterFiles),
+            OuterChecksum = checksum(Tarball),
+
+            case valid_size(Tarball, TarballMaxSize) of
+                true ->
+                    {ok, #{
+                        tarball => Tarball,
+                        outer_checksum => OuterChecksum,
+                        inner_checksum => InnerChecksum
+                    }};
+                false ->
+                    {error, {tarball, {too_big_compressed, TarballMaxSize}}}
+            end;
+        false ->
+            {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}}
     end.
 
 -spec create(metadata(), files()) ->
@@ -111,21 +119,26 @@ create(Metadata, Files) ->
 %% '''
 %% @end
 -spec create_docs(files(), hex_core:config()) -> {ok, tarball()} | {error, term()}.
-create_docs(Files, #{
-    tarball_max_size := TarballMaxSize, tarball_max_uncompressed_size := TarballMaxUncompressedSize
-}) ->
+create_docs(Files, Config) ->
+    #{
+        docs_tarball_max_size := TarballMaxSize,
+        docs_tarball_max_uncompressed_size := TarballMaxUncompressedSize
+    } = Config,
+
     UncompressedTarball = create_memory_tarball(Files),
-    UncompressedSize = byte_size(UncompressedTarball),
-    Tarball = gzip(UncompressedTarball),
-    Size = byte_size(Tarball),
-
-    case {(Size > TarballMaxSize), (UncompressedSize > TarballMaxUncompressedSize)} of
-        {_, true} ->
-            {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}};
-        {true, _} ->
-            {error, {tarball, {too_big_compressed, TarballMaxSize}}};
-        {false, false} ->
-            {ok, Tarball}
+
+    case valid_size(UncompressedTarball, TarballMaxUncompressedSize) of
+        true ->
+            Tarball = gzip(UncompressedTarball),
+
+            case valid_size(Tarball, TarballMaxSize) of
+                true ->
+                    {ok, Tarball};
+                false ->
+                    {error, {tarball, {too_big_compressed, TarballMaxSize}}}
+            end;
+        false ->
+            {error, {tarball, {too_big_uncompressed, TarballMaxUncompressedSize}}}
     end.
 
 -spec create_docs(files()) -> {ok, tarball()}.
@@ -166,19 +179,20 @@ create_docs(Files) ->
             metadata => metadata()
         }}
         | {error, term()}.
-unpack(Tarball, _, #{tarball_max_size := TarballMaxSize}) when
-    byte_size(Tarball) > TarballMaxSize
-->
-    {error, {tarball, too_big}};
-unpack(Tarball, Output, _Config) ->
-    case hex_erl_tar:extract({binary, Tarball}, [memory]) of
-        {ok, []} ->
-            {error, {tarball, empty}};
-        {ok, FileList} ->
-            OuterChecksum = crypto:hash(sha256, Tarball),
-            do_unpack(maps:from_list(FileList), OuterChecksum, Output);
-        {error, Reason} ->
-            {error, {tarball, Reason}}
+unpack(Tarball, Output, Config) ->
+    case valid_size(Tarball, maps:get(tarball_max_size, Config)) of
+        true ->
+            case hex_erl_tar:extract({binary, Tarball}, [memory]) of
+                {ok, []} ->
+                    {error, {tarball, empty}};
+                {ok, FileList} ->
+                    OuterChecksum = crypto:hash(sha256, Tarball),
+                    do_unpack(maps:from_list(FileList), OuterChecksum, Output);
+                {error, Reason} ->
+                    {error, {tarball, Reason}}
+            end;
+        false ->
+            {error, {tarball, too_big}}
     end.
 
 %% @doc
@@ -219,12 +233,13 @@ unpack(Tarball, Output) ->
 -spec unpack_docs
     (tarball(), memory, hex_core:config()) -> {ok, contents()} | {error, term()};
     (tarball(), filename(), hex_core:config()) -> ok | {error, term()}.
-unpack_docs(Tarball, _, #{tarball_max_size := TarballMaxSize}) when
-    byte_size(Tarball) > TarballMaxSize
-->
-    {error, {tarball, too_big}};
-unpack_docs(Tarball, Output, _Config) ->
-    unpack_tarball(Tarball, Output).
+unpack_docs(Tarball, Output, Config) ->
+    case valid_size(Tarball, maps:get(docs_tarball_max_size, Config)) of
+        true ->
+            unpack_tarball(Tarball, Output);
+        false ->
+            {error, {tarball, too_big}}
+    end.
 
 -spec unpack_docs
     (tarball(), memory) -> {ok, contents()} | {error, term()};
@@ -601,6 +616,12 @@ gzip_no_header(Uncompressed) ->
 %% Helpers
 %%====================================================================
 
+%% @private
+valid_size(Binary, infinity) when is_binary(Binary) ->
+    true;
+valid_size(Binary, Limit) when is_binary(Binary) and is_integer(Limit) ->
+    byte_size(Binary) =< Limit.
+
 %% @private
 binarify(Binary) when is_binary(Binary) -> Binary;
 binarify(Number) when is_number(Number) -> Number;

diff --git a/test/hex_tarball_SUITE.erl b/test/hex_tarball_SUITE.erl
@@ -328,17 +328,17 @@ unpack_error_handling_test(_Config) ->
 
 docs_too_big_to_create_test(_Config) ->
     Files = [{"index.html", <<"Docs">>}],
-    Config = maps:put(tarball_max_size, 100, hex_core:default_config()),
+    Config = maps:put(docs_tarball_max_size, 100, hex_core:default_config()),
     {error, {tarball, {too_big_compressed, 100}}} = hex_tarball:create_docs(Files, Config),
-    Config1 = maps:put(tarball_max_uncompressed_size, 100, hex_core:default_config()),
+    Config1 = maps:put(docs_tarball_max_uncompressed_size, 100, hex_core:default_config()),
     {error, {tarball, {too_big_uncompressed, 100}}} = hex_tarball:create_docs(Files, Config1),
 
     ok.
 
 docs_too_big_to_unpack_test(_Config) ->
     Files = [{"index.html", <<"Docs">>}],
     {ok, Tarball} = hex_tarball:create_docs(Files),
-    Config = maps:put(tarball_max_size, 100, hex_core:default_config()),
+    Config = maps:put(docs_tarball_max_size, 100, hex_core:default_config()),
     {error, {tarball, too_big}} = hex_tarball:unpack_docs(Tarball, memory, Config),
 
     ok.