Skip to content

hex_taball to be able to unpack list of files #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
RudolfMan wants to merge 2 commits into from
Closed

hex_taball to be able to unpack list of files #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8295d4b
add hex_tarball:unpack/3 that accepts list of files to unpack
RudolfMan May 1, 2020
7886b9c
Merge branch 'master' into unpack-list-of-files and resolve conflicts
RudolfMan Jun 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,14 @@ Unpack package tarball:
{ok, #{outer_checksum := Checksum, contents := Contents, metadata := Metadata}} = hex_tarball:unpack(Tarball, memory).
```

Or provide the list of files you want to extract from the tarball

```erlang
File = "dir/foo",
{ok, #{outer_checksum := Checksum, contents := [{File, Content}], metadata := Metadata}} = hex_tarball:unpack(Tarball, [File], memory).
```


Remember to verify the outer tarball checksum against the registry checksum
returned from `hex_repo:get_package(Config, Package)`.

Expand Down
58 changes: 42 additions & 16 deletions src/hex_tarball.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-module(hex_tarball).
-export([create/2, create/3, create_docs/1, create_docs/2, unpack/2, unpack/3,
unpack_docs/2, unpack_docs/3, format_checksum/1, format_error/1]).
unpack/4, unpack_docs/2, unpack_docs/3, format_checksum/1, format_error/1]).
-ifdef(TEST).
-export([do_decode_metadata/1, gzip/1, normalize_requirements/1]).
-endif.
Expand Down Expand Up @@ -122,34 +122,56 @@ create_docs(Files) ->
%% contents => [{"src/foo.erl",<<"-module(foo).">>}],
%% metadata => #{<<"name">> => <<"foo">>, ...}}}
%%
%% > hex_tarball:unpack(Tarball, ["src/foo", "src/bar"], memory).
%% {ok,#{outer_checksum => <<...>>,
%% contents => [{"src/foo",<<...>>},
%% {"src/bar",<<...>>}],
%% metadata => #{<<"name">> => <<"foo">>, ...}}}
%%
%% > hex_tarball:unpack(Tarball, "path/to/unpack").
%% {ok,#{outer_checksum => <<...>>,
%% metadata => #{<<"name">> => <<"foo">>, ...}}}
%% '''
-spec unpack(tarball(), memory, hex_core:config()) ->
-spec unpack(tarball(), [filename()] | all_files, memory, hex_core:config()) ->
{ok, #{outer_checksum => checksum(), inner_checksum => checksum(),
metadata => metadata(), contents => contents()}} |
{error, term()};
(tarball(), filename(), hex_core:config()) ->
(tarball(), [filename()] | all_files, filename(), hex_core:config()) ->
{ok, #{outer_checksum => checksum(), inner_checksum => checksum(),
metadata => metadata()}} |
{error, term()}.
unpack(Tarball, _, #{tarball_max_size := TarballMaxSize}) when byte_size(Tarball) > TarballMaxSize ->
unpack(Tarball, _, _, #{tarball_max_size := TarballMaxSize}) when byte_size(Tarball) > TarballMaxSize ->
{error, {tarball, too_big}};

unpack(Tarball, Output, _Config) ->

unpack(Tarball, ListOfFilesToExtract, Output, _Config) ->
case hex_erl_tar:extract({binary, Tarball}, [memory]) of
{ok, []} ->
{error, {tarball, empty}};

{ok, FileList} ->
OuterChecksum = crypto:hash(sha256, Tarball),
do_unpack(maps:from_list(FileList), OuterChecksum, Output);
Opts = case ListOfFilesToExtract of
all_files -> [];
_ -> [{files, ListOfFilesToExtract}]
end,
do_unpack(maps:from_list(FileList), OuterChecksum, Output, Opts);

{error, Reason} ->
{error, {tarball, Reason}}
end.

-spec unpack(tarball(), [filename()] | all_files, memory) ->
{ok, #{outer_checksum => checksum(), inner_checksum => checksum(),
metadata => metadata(), contents => contents()}} |
{error, term()};
(tarball(), [filename()] | all_files, filename()) ->
{ok, #{outer_checksum => checksum(), inner_checksum => checksum(),
metadata => metadata()}} |
{error, term()}.
unpack(Tarball, ListOfFilesToExtract, Output) ->
unpack(Tarball, ListOfFilesToExtract, Output, hex_core:default_config()).


-spec unpack(tarball(), memory) ->
{ok, #{outer_checksum => checksum(), inner_checksum => checksum(),
Expand All @@ -160,7 +182,7 @@ unpack(Tarball, Output, _Config) ->
metadata => metadata()}} |
{error, term()}.
unpack(Tarball, Output) ->
unpack(Tarball, Output, hex_core:default_config()).
unpack(Tarball, all_files, Output, hex_core:default_config()).

%% @doc
%% Unpacks a documentation tarball.
Expand Down Expand Up @@ -233,7 +255,7 @@ encode_metadata(Meta) ->
end, maps:to_list(Meta)),
iolist_to_binary(Data).

do_unpack(Files, OuterChecksum, Output) ->
do_unpack(Files, OuterChecksum, Output, Opts) ->
State = #{
inner_checksum => undefined,
outer_checksum => OuterChecksum,
Expand All @@ -246,15 +268,16 @@ do_unpack(Files, OuterChecksum, Output) ->
State2 = check_version(State1),
State3 = check_inner_checksum(State2),
State4 = decode_metadata(State3),
finish_unpack(State4).
finish_unpack(State4, Opts).

finish_unpack({error, _} = Error) ->
finish_unpack({error, _} = Error, _) ->
Error;
finish_unpack(#{metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output}) ->

finish_unpack(#{metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output}, Opts) ->
_ = maps:get("VERSION", Files),
ContentsBinary = maps:get("contents.tar.gz", Files),
filelib:ensure_dir(filename:join(Output, "*")),
case unpack_tarball(ContentsBinary, Output) of
case unpack_tarball(ContentsBinary, Output, Opts) of
ok ->
copy_metadata_config(Output, maps:get("metadata.config", Files)),
{ok, #{inner_checksum => InnerChecksum, outer_checksum => OuterChecksum, metadata => Metadata}};
Expand Down Expand Up @@ -387,12 +410,15 @@ guess_build_tools(Metadata) ->
%%====================================================================
%% Tar Helpers
%%====================================================================

unpack_tarball(ContentsBinary, memory) ->
hex_erl_tar:extract({binary, ContentsBinary}, [memory, compressed]);
unpack_tarball(ContentsBinary, Output) ->
unpack_tarball(ContentsBinary, Output, []).

unpack_tarball(ContentsBinary, memory, Opts) ->
hex_erl_tar:extract({binary, ContentsBinary}, [memory, compressed | Opts]);

unpack_tarball(ContentsBinary, Output, Opts) ->
filelib:ensure_dir(filename:join(Output, "*")),
case hex_erl_tar:extract({binary, ContentsBinary}, [{cwd, Output}, compressed]) of
case hex_erl_tar:extract({binary, ContentsBinary}, [{cwd, Output}, compressed | Opts]) of
ok ->
[try_updating_mtime(filename:join(Output, Path)) || Path <- filelib:wildcard("**", Output)],
ok;
Expand Down
23 changes: 21 additions & 2 deletions test/hex_tarball_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ all() ->
memory_test, build_tools_test, requirements_test,
decode_metadata_test, unpack_error_handling_test,
docs_test, too_big_to_create_test, too_big_to_unpack_test,
docs_too_big_to_create_test, docs_too_big_to_unpack_test
docs_too_big_to_create_test, docs_too_big_to_unpack_test,
unpack_all_files_test, unpack_list_of_files_test
].

too_big_to_create_test(_Config) ->
Expand All @@ -36,7 +37,7 @@ too_big_to_unpack_test(_Config) ->
Contents = [{"src/foo.erl", <<"-module(foo).">>}],
{ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, Contents),
Config = maps:put(tarball_max_size, 5100, hex_core:default_config()),
{error, {tarball, too_big}} = hex_tarball:unpack(Tarball, memory, Config),
{error, {tarball, too_big}} = hex_tarball:unpack(Tarball, all_files, memory, Config),
ok.

memory_test(_Config) ->
Expand Down Expand Up @@ -281,6 +282,24 @@ docs_test(Config) ->

ok.

unpack_all_files_test(_Config) ->
Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>},
Files = [{"foo", <<"">>},{"bar", <<"">>}],
{ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, Files),

{ok, #{contents := Files}} = hex_tarball:unpack(Tarball, memory),

ok.

unpack_list_of_files_test(_Config) ->
Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>},
ExpectedFile = {"foo", <<"">>},
{ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, [ExpectedFile, {"bar", <<"">>}]),

{ok, #{contents := [ExpectedFile]}} = hex_tarball:unpack(Tarball, ["foo"], memory),

ok.

%%====================================================================
%% Helpers
%%====================================================================
Expand Down