diff --git a/.env b/.env index a8e263d..e6462af 100644 --- a/.env +++ b/.env @@ -2,6 +2,6 @@ # You SHOULD specify point releases here so that build time and run time Erlang/OTPs # are the same. See: https://github.com/erlware/relx/pull/902 SERVICE_NAME=url-shortener -OTP_VERSION=27.1.2 -REBAR_VERSION=3.24 +OTP_VERSION=28.5.0 +REBAR_VERSION=3.26 THRIFT_VERSION=0.14.2.3 diff --git a/apps/shortener/src/shortener_handler.erl b/apps/shortener/src/shortener_handler.erl index b7c401b..b636593 100644 --- a/apps/shortener/src/shortener_handler.erl +++ b/apps/shortener/src/shortener_handler.erl @@ -243,20 +243,31 @@ construct_shortened_url( ) -> #{ <<"id">> => ID, - <<"shortenedUrl">> => render_short_url(ID, get_short_url_template()), + <<"shortenedUrl">> => render_short_url(ID, get_short_url_template(), Source), <<"sourceUrl">> => Source, <<"expiresAt">> => ExpiresAt }. -render_short_url(ID, Template) -> +render_short_url(ID, Template, Source) -> + Netloc = genlib:to_binary(maps:get(netloc, Template)), + AuthorityMapper = maps:get(authority_mapper, Template, undefined), iolist_to_binary([ genlib:to_binary(maps:get(scheme, Template)), <<"://">>, - genlib:to_binary(maps:get(netloc, Template)), + render_short_authority(Source, AuthorityMapper, Netloc), genlib:to_binary(maps:get(path, Template)), ID ]). +render_short_authority(_Source, undefined, Netloc) -> + Netloc; +render_short_authority(Source, auto, _Netloc) -> + #{host := SrcDomain} = uri_string:parse(Source), + unicode:characters_to_binary(SrcDomain); +render_short_authority(Source, MapRules, Netloc) -> + #{host := SrcDomain} = uri_string:parse(Source), + unicode:characters_to_binary(maps:get(unicode:characters_to_list(SrcDomain), MapRules, Netloc)). + parse_timestamp(Timestamp) -> Microseconds = genlib_rfc3339:parse(Timestamp, microsecond), genlib_rfc3339:format_relaxed(Microseconds, microsecond). diff --git a/apps/shortener/test/shortener_ct_helper.erl b/apps/shortener/test/shortener_ct_helper.erl index b600809..68938bd 100644 --- a/apps/shortener/test/shortener_ct_helper.erl +++ b/apps/shortener/test/shortener_ct_helper.erl @@ -12,6 +12,7 @@ -export([get_app_config/2]). -export([get_app_config/3]). -export([get_app_config/4]). +-export([get_app_config/5]). -export([get_bouncer_client_app_config/0]). -type config() :: [{atom(), any()}]. @@ -108,6 +109,10 @@ get_app_config(Port, Netloc, AutomatonUrl) -> -spec get_app_config(_, _, _, _) -> _. get_app_config(Port, Netloc, AutomatonUrl, MachineryBackend) -> + get_app_config(Port, Netloc, undefined, AutomatonUrl, MachineryBackend). + +-spec get_app_config(_, _, _, _, _) -> _. +get_app_config(Port, Netloc, AuthorityMapper, AutomatonUrl, MachineryBackend) -> [ {machinery_backend, MachineryBackend}, {bouncer_ruleset_id, <<"service/authz/api">>}, @@ -123,6 +128,7 @@ get_app_config(Port, Netloc, AutomatonUrl, MachineryBackend) -> ], short_url_template => #{ scheme => http, + authority_mapper => AuthorityMapper, netloc => Netloc, path => "/r/e/d/i/r/" } diff --git a/apps/shortener/test/shortener_general_SUITE.erl b/apps/shortener/test/shortener_general_SUITE.erl index fe16fac..e6243cc 100644 --- a/apps/shortener/test/shortener_general_SUITE.erl +++ b/apps/shortener/test/shortener_general_SUITE.erl @@ -40,14 +40,22 @@ -spec all() -> [test_case_name() | {group, group_name()}]. all() -> [ - {group, general}, - {group, cors}, + {group, single_domain}, + {group, multi_domain}, {group, misc} ]. -spec groups() -> [{atom(), list(), [test_case_name()]}]. groups() -> [ + {multi_domain, [], [ + {group, general}, + {group, cors} + ]}, + {single_domain, [], [ + {group, general}, + {group, cors} + ]}, {general, [], [ successful_redirect, successful_delete, @@ -92,34 +100,58 @@ init_per_suite(C) -> ] ++ C. -spec init_per_group(atom(), config()) -> config(). -init_per_group(misc, C) -> +init_per_group(single_domain, C) -> ShortenerApp = genlib_app:start_application_with( shortener, shortener_ct_helper:get_app_config( ?config(port, C), ?config(netloc, C), - <<"http://invalid_url:8022/v1/automaton">>, + <<"http://machinegun:8022/v1/automaton">>, progressor ) ), [{shortener_app, ShortenerApp}] ++ C; -init_per_group(_Group, C) -> +init_per_group(multi_domain, C) -> ShortenerApp = genlib_app:start_application_with( shortener, shortener_ct_helper:get_app_config( ?config(port, C), ?config(netloc, C), + #{ + "oops.io" => "oops.io:8080", + "example.com" => "example.com:8080" + }, <<"http://machinegun:8022/v1/automaton">>, progressor ) ), - [{shortener_app, ShortenerApp}] ++ C. + [{shortener_app, ShortenerApp}] ++ C; +init_per_group(misc, C) -> + ShortenerApp = + genlib_app:start_application_with( + shortener, + shortener_ct_helper:get_app_config( + ?config(port, C), + ?config(netloc, C), + <<"http://invalid_url:8022/v1/automaton">>, + progressor + ) + ), + [{shortener_app, ShortenerApp}] ++ C; +init_per_group(_Group, C) -> + C. -spec end_per_group(atom(), config()) -> _. -end_per_group(_Group, C) -> - genlib_app:stop_unload_applications(?config(shortener_app, C)). +end_per_group(Group, C) when + Group =:= single_domain; + Group =:= multi_domain; + Group =:= misc +-> + genlib_app:stop_unload_applications(?config(shortener_app, C)); +end_per_group(_Group, _C) -> + ok. -spec end_per_suite(config()) -> term(). end_per_suite(C) -> diff --git a/compose.yaml b/compose.yaml index 649fbe7..82f4891 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,9 @@ services: condition: service_healthy ports: - "8022:8022" + extra_hosts: + - "oops.io:127.0.0.1" + - "example.com:127.0.0.1" command: /sbin/init machinegun: @@ -41,8 +44,6 @@ services: POSTGRES_USER: "progressor" POSTGRES_PASSWORD: "progressor" PGDATA: "/tmp/postgresql/data/pgdata" - volumes: - - progressor-data:/tmp/postgresql/data ports: - "5432:5432" healthcheck: @@ -57,6 +58,3 @@ services: limits: cpus: '1' memory: 4G - -volumes: - progressor-data: diff --git a/config/sys.config b/config/sys.config index dcccd45..9c040cc 100644 --- a/config/sys.config +++ b/config/sys.config @@ -14,6 +14,14 @@ port => 8080, short_url_template => #{ scheme => https, + %% If a authority_mapper is configured, the netloc parameter will be used as default shortened destination authority + authority_mapper => #{ + %% source url domain(host) %% shortened destination authority (port is optional) + "28-may-vlty.zone" => "vality.dev:8022" + }, + %% mapper supports 'auto' mode. In this case, the source URL will be used to create a short link. + %% "auto" mode can only be used on standard ports + %% authority_mapper => auto, netloc => "rbk.mn", path => "/" }, diff --git a/rebar.config b/rebar.config index e58738a..fbf0672 100644 --- a/rebar.config +++ b/rebar.config @@ -26,13 +26,13 @@ % Common project dependencies. {deps, [ - {cowboy, "2.9.0"}, + {cowboy, "2.12.0"}, {genlib, {git, "https://github.com/valitydev/genlib.git", {tag, "v1.1.0"}}}, {cowboy_access_log, {git, "https://github.com/valitydev/cowboy_access_log.git", {branch, "master"}}}, - {woody, {git, "https://github.com/valitydev/woody_erlang.git", {tag, "v1.1.0"}}}, + {woody, {git, "https://github.com/valitydev/woody_erlang.git", {tag, "v1.1.2"}}}, {woody_user_identity, {git, "https://github.com/valitydev/woody_erlang_user_identity.git", {tag, "v1.1.0"}}}, {scoper, {git, "https://github.com/valitydev/scoper.git", {tag, "v1.1.0"}}}, - {machinery, {git, "https://github.com/valitydev/machinery-erlang.git", {tag, "v1.1.10"}}}, + {machinery, {git, "https://github.com/valitydev/machinery-erlang.git", {tag, "v1.1.21"}}}, {bouncer_client, {git, "https://github.com/valitydev/bouncer-client-erlang.git", {tag, "v1.1.0"}}}, {token_keeper_client, {git, "https://github.com/valitydev/token-keeper-client.git", {tag, "v1.1.0"}}}, {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index 3016b87..b11bab1 100644 --- a/rebar.lock +++ b/rebar.lock @@ -21,7 +21,7 @@ {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}}, 1}, {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.15.1">>},2}, - {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},0}, + {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.12.0">>},0}, {<<"cowboy_access_log">>, {git,"https://github.com/valitydev/cowboy_access_log.git", {ref,"04da359e022cf05c5c93812504d5791d6bc97453"}}, @@ -30,13 +30,17 @@ {git,"https://github.com/valitydev/cowboy_cors.git", {ref,"5a3b084fb8c5a4ff58e3c915a822d709d6023c3b"}}, 0}, - {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},1}, + {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.13.0">>},1}, {<<"crc32cer">>,{pkg,<<"crc32cer">>,<<"0.1.11">>},4}, {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2}, {<<"email_validator">>,{pkg,<<"email_validator">>,<<"1.1.0">>},0}, {<<"epg_connector">>, {git,"https://github.com/valitydev/epg_connector.git", - {ref,"2e86da8083908d0d35a4eed3e2168c9ba6a8d04a"}}, + {ref,"939a0d4ab3f7561a79b45381bbe13029d9263006"}}, + 2}, + {<<"epg_migrator">>, + {git,"https://github.com/valitydev/epg_migrator.git", + {ref,"8633c43fb4d022355c1498c0effa9ecf4098a67f"}}, 2}, {<<"epgsql">>, {git,"https://github.com/epgsql/epgsql.git", @@ -46,6 +50,7 @@ {git,"https://github.com/valitydev/erlang-health.git", {ref,"49716470d0e8dab5e37db55d52dea78001735a3d"}}, 0}, + {<<"erlydtl">>,{pkg,<<"erlydtl">>,<<"0.14.0">>},3}, {<<"genlib">>, {git,"https://github.com/valitydev/genlib.git", {ref,"d2324089afbbd9630e85fac554620f1de0b33dfe"}}, @@ -72,7 +77,7 @@ {<<"kafka_protocol">>,{pkg,<<"kafka_protocol">>,<<"4.1.10">>},3}, {<<"machinery">>, {git,"https://github.com/valitydev/machinery-erlang.git", - {ref,"066f6cecd5d8f011c1b3bf8e45f6ad967c0e733d"}}, + {ref,"24982b9ae38e029d23b724b0cbf7105c1ad978c9"}}, 0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},1}, {<<"mg_proto">>, @@ -92,7 +97,7 @@ {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},0}, {<<"progressor">>, {git,"https://github.com/valitydev/progressor.git", - {ref,"d429410a0f2b42fbcc22b340cc8dc1915b13f119"}}, + {ref,"90f46570007a1cbf7cac960059fe6c8018702489"}}, 1}, {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.11.0">>},0}, {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.9">>},0}, @@ -135,7 +140,7 @@ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},1}, {<<"woody">>, {git,"https://github.com/valitydev/woody_erlang.git", - {ref,"cc983a9423325ba1d6a509775eb6ff7ace721539"}}, + {ref,"7194b5d97fc2f68fffaf22c0278c7036b48dd11b"}}, 0}, {<<"woody_user_identity">>, {git,"https://github.com/valitydev/woody_erlang_user_identity.git", @@ -149,11 +154,12 @@ {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>}, {<<"certifi">>, <<"DBAB8E5E155A0763EEA978C913CA280A6B544BFA115633FA20249C3D396D9493">>}, {<<"chatterbox">>, <<"5CAC4D15DD7AD61FC3C4415CE4826FC563D4643DEE897A558EC4EA0B1C835C9C">>}, - {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>}, - {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>}, + {<<"cowboy">>, <<"F276D521A1FF88B2B9B4C54D0E753DA6C66DD7BE6C9FCA3D9418B561828A3731">>}, + {<<"cowlib">>, <<"DB8F7505D8332D98EF50A3EF34B34C1AFDDEC7506E4EE4DD4A3A266285D282CA">>}, {<<"crc32cer">>, <<"B550DA6D615FEB72A882D15D020F8F7DEE72DFB2CB1BCDF3B1EE8DC2AFD68CFC">>}, {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>}, {<<"email_validator">>, <<"7E09A862E9AA99AE2CA6FD2A718D2B94360E32940A1339B53DFEE6B774BCDB03">>}, + {<<"erlydtl">>, <<"964B2DC84F8C17ACFAA69C59BA129EF26AC45D2BA898C3C6AD9B5BDC8BA13CED">>}, {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>}, {<<"grpcbox">>, <<"6E040AB3EF16FE699FFB513B0EF8E2E896DA7B18931A1EF817143037C454BCCE">>}, {<<"hackney">>, <<"99DA4674592504D3FB0CFEF0DB84C3BA02B4508BAE2DFF8C0108BAA0D6E0977C">>}, @@ -184,11 +190,12 @@ {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>}, {<<"certifi">>, <<"524C97B4991B3849DD5C17A631223896272C6B0AF446778BA4675A1DFF53BB7E">>}, {<<"chatterbox">>, <<"4F75B91451338BC0DA5F52F3480FA6EF6E3A2AEECFC33686D6B3D0A0948F31AA">>}, - {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>}, - {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>}, + {<<"cowboy">>, <<"8A7ABE6D183372CEB21CAA2709BEC928AB2B72E18A3911AA1771639BEF82651E">>}, + {<<"cowlib">>, <<"E1E1284DC3FC030A64B1AD0D8382AE7E99DA46C3246B815318A4B848873800A4">>}, {<<"crc32cer">>, <<"A39B8F0B1990AC1BF06C3A247FC6A178B740CDFC33C3B53688DC7DD6B1855942">>}, {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>}, {<<"email_validator">>, <<"2B1E6DF7BB14155C8D7D131F1C95CF4676200BC056EEBA82123396833FF94DA2">>}, + {<<"erlydtl">>, <<"D80EC044CD8F58809C19D29AC5605BE09E955040911B644505E31E9DD8143431">>}, {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>}, {<<"grpcbox">>, <<"4A3B5D7111DAABC569DC9CBD9B202A3237D81C80BF97212FBC676832CB0CEB17">>}, {<<"hackney">>, <<"DE16FF4996556C8548D512F4DBE22DD58A587BF3332E7FD362430A7EF3986B16">>},