jacob.eva
/
ejabberd-contrib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'http-upload' 

* http-upload:
  mod_http_upload: Use application/octet-stream instead of text/html for security reasons
  mod_http_upload: Add note regarding "get_url"
  mod_http_upload: Use full PUT URL for process name
  mod_http_upload: Rename variable for clarity
  mod_http_upload: Fix logging of file size mismatch
  mod_http_upload: Return error atoms, not strings
  mod_http_upload: Log message if URL looks wrong
  mod_http_upload_quota: Don't track all users
  mod_http_upload: Make case indentation consistent
  mod_http_upload: Apply cosmetic changes
  mod_http_upload: Don't flatten command strings
  mod_http_upload: Don't let identify/1 return error
  mod_http_upload_quota: Omit info message
  mod_http_upload: Don't complain during shutdown
  mod_http_upload: Use binary strings in most places
  mod_http_upload: Check availability of ImageMagick
  mod_http_upload: Suppress error on non-image files
  mod_http_upload: Fix string()/binary() type issue
  mod_http_upload: Add/fix function specifications
  mod_http_upload: Strip newline from command output
  mod_http_upload: Support thumbnail generation
This commit is contained in:
Holger Weiss 2016-01-12 00:05:05 +01:00
parent 0d2fa84c8a 4deb2fd2c3
commit d73b648cf3
3 changed files with 472 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mod_http_upload/README.txt
View File

@ -84,6 +84,13 @@ The configurable mod_http_upload options are:
mod_http_upload. Otherwise, a SHA-1 hash of the user's bare JID is
included instead.
- thumbnail: (default: 'true')
This option specifies whether ejabberd should create thumbnails of
uploaded images. If a thumbnail is created, a <thumbnail/> element that
contains the download <uri/> and some metadata is returned with the PUT
response.
- file_mode (default: 'undefined')
This option defines the permission bits of uploaded files. The bits are
@ -108,7 +115,7 @@ The configurable mod_http_upload options are:
This option specifies the initial part of the PUT URLs used for file
uploads. The keyword @HOST@ is replaced with the virtual host name.
NOTE: Different virtual hosts cannot use the same PUT URL domain.
NOTE: Different virtual hosts cannot use the same PUT URL.
- get_url (default: $put_url)
@ -118,6 +125,9 @@ The configurable mod_http_upload options are:
files served by a proper HTTP server such as Nginx or Apache. The keyword
@HOST@ is replaced with the virtual host name.
NOTE: If GET requests are handled by mod_http_upload, the 'get_url' must
match the 'put_url'.
- service_url (default: 'undefined')
If a 'service_url' is specified, HTTP upload slot requests are forwarded

222
mod_http_upload/src/mod_http_upload.erl
View File

@ -25,7 +25,6 @@
{<<".bz2">>, <<"application/x-bzip2">>},
{<<".gif">>, <<"image/gif">>},
{<<".gz">>, <<"application/x-gzip">>},
{<<".html">>, <<"text/html">>},
{<<".jpeg">>, <<"image/jpeg">>},
{<<".jpg">>, <<"image/jpeg">>},
{<<".mp3">>, <<"audio/mpeg">>},
@ -90,10 +89,17 @@
put_url :: binary(),
get_url :: binary(),
service_url :: binary() | undefined,
thumbnail :: boolean(),
slots = dict:new() :: term()}). % dict:dict() requires Erlang 17.
-record(media_info,
{type :: binary(),
height :: integer(),
width :: integer()}).
-type state() :: #state{}.
-type slot() :: [binary()].
-type slot() :: [binary(), ...].
-type media_info() :: #media_info{}.
%%--------------------------------------------------------------------
%% gen_mod/supervisor callbacks.
@ -116,7 +122,8 @@ start(ServerHost, Opts) ->
remove_user, 50),
ejabberd_hooks:add(anonymous_purge_hook, ServerHost, ?MODULE,
remove_user, 50);
false -> ok
false ->
ok
end,
Proc = get_proc_name(ServerHost, ?PROCNAME),
Spec = {Proc,
@ -138,11 +145,12 @@ stop(ServerHost) ->
remove_user, 50),
ejabberd_hooks:delete(anonymous_purge_hook, ServerHost, ?MODULE,
remove_user, 50);
false -> ok
false ->
ok
end,
Proc = get_proc_name(ServerHost, ?PROCNAME),
ok = supervisor:terminate_child(ejabberd_sup, Proc),
ok = supervisor:delete_child(ejabberd_sup, Proc).
supervisor:terminate_child(ejabberd_sup, Proc),
supervisor:delete_child(ejabberd_sup, Proc).
-spec mod_opt_type(atom()) -> fun((term()) -> term()) | [atom()].
@ -188,10 +196,12 @@ mod_opt_type(custom_headers) ->
end;
mod_opt_type(rm_on_unregister) ->
fun(B) when is_boolean(B) -> B end;
mod_opt_type(thumbnail) ->
fun(B) when is_boolean(B) -> B end;
mod_opt_type(_) ->
[host, name, access, max_size, secret_length, jid_in_url, file_mode,
dir_mode, docroot, put_url, get_url, service_url, custom_headers,
rm_on_unregister].
rm_on_unregister, thumbnail].
%%--------------------------------------------------------------------
%% gen_server callbacks.
@ -242,6 +252,9 @@ init({ServerHost, Opts}) ->
fun(<<"http://", _/binary>> = URL) -> URL;
(<<"https://", _/binary>> = URL) -> URL
end),
Thumbnail = gen_mod:get_opt(thumbnail, Opts,
fun(B) when is_boolean(B) -> B end,
true),
case ServiceURL of
undefined ->
ok;
@ -260,11 +273,24 @@ init({ServerHost, Opts}) ->
Mode ->
file:change_mode(DocRoot, Mode)
end,
case Thumbnail of
true ->
case string:str(os:cmd("identify"), "Magick") of
0 ->
?ERROR_MSG("Cannot find 'identify' command, please install "
"ImageMagick or disable thumbnail creation", []);
_ ->
ok
end;
false ->
ok
end,
ejabberd_router:register_route(Host),
{ok, #state{server_host = ServerHost, host = Host, name = Name,
access = Access, max_size = MaxSize,
secret_length = SecretLength, jid_in_url = JIDinURL,
file_mode = FileMode, dir_mode = DirMode,
thumbnail = Thumbnail,
docroot = expand_home(str:strip(DocRoot, right, $/)),
put_url = expand_host(str:strip(PutURL, right, $/), ServerHost),
get_url = expand_host(str:strip(GetURL, right, $/), ServerHost),
@ -278,13 +304,16 @@ init({ServerHost, Opts}) ->
handle_call({use_slot, Slot}, _From, #state{file_mode = FileMode,
dir_mode = DirMode,
get_url = GetPrefix,
thumbnail = Thumbnail,
docroot = DocRoot} = State) ->
case get_slot(Slot, State) of
{ok, {Size, Timer}} ->
timer:cancel(Timer),
NewState = del_slot(Slot, State),
Path = str:join([DocRoot | Slot], <<$/>>),
{reply, {ok, Size, Path, FileMode, DirMode}, NewState};
{reply, {ok, Size, Path, FileMode, DirMode, GetPrefix, Thumbnail},
NewState};
error ->
{reply, {error, <<"Invalid slot">>}, State}
end;
@ -345,22 +374,34 @@ code_change(_OldVsn, #state{server_host = ServerHost} = State, _Extra) ->
-spec process([binary()], #request{})
-> {pos_integer(), [{binary(), binary()}], binary()}.
process(LocalPath, #request{method = 'PUT', host = Host, ip = IP,
data = Data}) ->
Proc = gen_mod:get_module_proc(Host, ?PROCNAME),
case catch gen_server:call(Proc, {use_slot, LocalPath}) of
{ok, Size, Path, FileMode, DirMode} when byte_size(Data) == Size ->
process(LocalPath, #request{method = Method, host = Host, ip = IP})
when length(LocalPath) < 3,
Method == 'PUT' orelse
Method == 'GET' orelse
Method == 'HEAD' ->
?DEBUG("Rejecting ~s request from ~s for ~s: Too few path components",
[Method, ?ADDR_TO_STR(IP), Host]),
http_response(Host, 404);
process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP,
data = Data} = Request) ->
{Proc, Slot} = parse_http_request(Request),
case catch gen_server:call(Proc, {use_slot, Slot}) of
{ok, Size, Path, FileMode, DirMode, GetPrefix, Thumbnail}
when byte_size(Data) == Size ->
?DEBUG("Storing file from ~s for ~s: ~s",
[?ADDR_TO_STR(IP), Host, Path]),
case store_file(Path, Data, FileMode, DirMode) of
case store_file(Path, Data, FileMode, DirMode,
GetPrefix, Slot, Thumbnail) of
ok ->
http_response(Host, 201);
{ok, Headers, OutData} ->
http_response(Host, 201, Headers, OutData);
{error, Error} ->
?ERROR_MSG("Cannot store file ~s from ~s for ~s: ~p",
[Path, ?ADDR_TO_STR(IP), Host, Error]),
[Path, ?ADDR_TO_STR(IP), Host, ?FORMAT(Error)]),
http_response(Host, 500)
end;
{ok, Size, Path} ->
{ok, Size, Path, _FileMode, _DirMode, _GetPrefix, _Thumbnail} ->
?INFO_MSG("Rejecting file ~s from ~s for ~s: Size is ~B, not ~B",
[Path, ?ADDR_TO_STR(IP), Host, byte_size(Data), Size]),
http_response(Host, 413);
@ -373,17 +414,16 @@ process(LocalPath, #request{method = 'PUT', host = Host, ip = IP,
[?ADDR_TO_STR(IP), Host, Error]),
http_response(Host, 500)
end;
process(LocalPath, #request{method = Method, host = Host, ip = IP})
process(_LocalPath, #request{method = Method, host = Host, ip = IP} = Request)
when Method == 'GET';
Method == 'HEAD' ->
Proc = gen_mod:get_module_proc(Host, ?PROCNAME),
{Proc, [_UserDir, _RandDir, FileName] = Slot} = parse_http_request(Request),
case catch gen_server:call(Proc, get_docroot) of
{ok, DocRoot} ->
Path = str:join([DocRoot | LocalPath], <<$/>>),
Path = str:join([DocRoot | Slot], <<$/>>),
case file:read_file(Path) of
{ok, Data} ->
?INFO_MSG("Serving ~s to ~s", [Path, ?ADDR_TO_STR(IP)]),
FileName = lists:last(LocalPath),
ContentType = guess_content_type(FileName),
Headers1 = case ContentType of
<<"image/", _SubType/binary>> -> [];
@ -400,7 +440,7 @@ process(LocalPath, #request{method = Method, host = Host, ip = IP})
[Path, ?ADDR_TO_STR(IP)]),
http_response(Host, 403);
{error, enoent} ->
?INFO_MSG("Cannot serve ~s to ~s: No such file or directory",
?INFO_MSG("Cannot serve ~s to ~s: No such file",
[Path, ?ADDR_TO_STR(IP)]),
http_response(Host, 404);
{error, eisdir} ->
@ -439,10 +479,10 @@ get_proc_name(ServerHost, ModuleName) ->
(_) -> <<"http://@HOST@">>
end,
<<"http://@HOST@">>),
[_, ProcHost | _] = binary:split(expand_host(PutURL, ServerHost),
[<<"http://">>, <<"https://">>,
<<":">>, <<"/">>], [global]),
gen_mod:get_module_proc(ProcHost, ModuleName).
{ok, {_Scheme, _UserInfo, Host, _Port, Path, _Query}} =
http_uri:parse(binary_to_list(expand_host(PutURL, ServerHost))),
ProcPrefix = list_to_binary(string:strip(Host ++ Path, right, $/)),
gen_mod:get_module_proc(ProcPrefix, ModuleName).
-spec expand_home(binary()) -> binary().
@ -478,10 +518,12 @@ process_iq(From,
allow ->
case parse_request(SubEl, Lang) of
{ok, File, Size, ContentType} ->
case create_slot(State, From, File, Size, ContentType, Lang) of
case create_slot(State, From, File, Size, ContentType,
Lang) of
{ok, Slot} ->
{ok, Timer} = timer:send_after(?SLOT_TIMEOUT,
{slot_timed_out, Slot}),
{slot_timed_out,
Slot}),
NewState = add_slot(Slot, Size, Timer, State),
SlotEl = slot_el(Slot, State, XMLNS),
{IQ#iq{type = result, sub_el = [SlotEl]}, NewState};
@ -581,12 +623,13 @@ create_slot(#state{service_url = ServiceURL},
case httpc:request(get, {GetRequest, []}, HttpOptions, Options) of
{ok, {Code, Body}} when Code >= 200, Code =< 299 ->
case binary:split(Body, <<$\n>>, [global, trim]) of
[<<"http", _/binary>> = PutURL, <<"http", _/binary>> = GetURL] ->
[<<"http", _/binary>> = PutURL,
<<"http", _/binary>> = GetURL] ->
?INFO_MSG("Got HTTP upload slot for ~s (file: ~s)",
[jlib:jid_to_string(JID), File]),
{ok, PutURL, GetURL};
Lines ->
?ERROR_MSG("Cannot parse data received for ~s from <~s>: ~p",
?ERROR_MSG("Can't parse data received for ~s from <~s>: ~p",
[jlib:jid_to_string(JID), ServiceURL, Lines]),
{error, ?ERR_SERVICE_UNAVAILABLE}
end;
@ -701,12 +744,59 @@ iq_disco_info(Lang, Name) ->
%% HTTP request handling.
-spec store_file(file:filename_all(), binary(),
-spec parse_http_request(#request{}) -> {atom(), slot()}.
parse_http_request(#request{host = Host, path = Path}) ->
PrefixLength = length(Path) - 3,
{ProcURL, Slot} = if PrefixLength > 0 ->
Prefix = lists:sublist(Path, PrefixLength),
{str:join([Host | Prefix], $/),
lists:nthtail(PrefixLength, Path)};
true ->
{Host, Path}
end,
{gen_mod:get_module_proc(ProcURL, ?PROCNAME), Slot}.
-spec store_file(binary(), binary(),
integer() | undefined,
integer() | undefined,
binary(), slot(), boolean())
-> ok | {ok, [{binary(), binary()}], binary()} | {error, term()}.
store_file(Path, Data, FileMode, DirMode, GetPrefix, Slot, Thumbnail) ->
case do_store_file(Path, Data, FileMode, DirMode) of
ok when Thumbnail ->
case identify(Path) of
{ok, MediaInfo} ->
case convert(Path, MediaInfo) of
{ok, OutPath} ->
[UserDir, RandDir | _] = Slot,
FileName = filename:basename(OutPath),
URL = str:join([GetPrefix, UserDir,
RandDir, FileName], <<$/>>),
ThumbEl = thumb_el(OutPath, URL),
{ok,
[{<<"Content-Type">>,
<<"text/xml; charset=utf-8">>}],
xml:element_to_binary(ThumbEl)};
pass ->
ok
end;
pass ->
ok
end;
ok ->
ok;
Err ->
Err
end.
-spec do_store_file(file:filename_all(), binary(),
integer() | undefined,
integer() | undefined)
-> ok | {error, term()}.
store_file(Path, Data, FileMode, DirMode) ->
do_store_file(Path, Data, FileMode, DirMode) ->
try
ok = filelib:ensure_dir(Path),
{ok, Io} = file:open(Path, [write, exclusive, raw]),
@ -728,7 +818,7 @@ store_file(Path, Data, FileMode, DirMode) ->
ok = Ok % Raise an exception if file:write/2 failed.
catch
_:{badmatch, {error, Error}} ->
{error, ?FORMAT(Error)};
{error, Error};
_:Error ->
{error, Error}
end.
@ -786,6 +876,70 @@ code_to_message(413) -> <<"File size doesn't match requested size.">>;
code_to_message(500) -> <<"Internal server error.">>;
code_to_message(_Code) -> <<"">>.
%%--------------------------------------------------------------------
%% Image manipulation stuff.
%%--------------------------------------------------------------------
-spec identify(binary()) -> {ok, media_info()} | pass.
identify(Path) ->
Cmd = io_lib:format("identify -format 'ok %m %h %w' ~s", [Path]),
Res = string:strip(os:cmd(Cmd), right, $\n),
case string:tokens(Res, " ") of
["ok", T, H, W] ->
{ok, #media_info{type = list_to_binary(string:to_lower(T)),
height = list_to_integer(H),
width = list_to_integer(W)}};
_ ->
?DEBUG("Cannot identify type of ~s: ~s", [Path, Res]),
pass
end.
-spec convert(binary(), media_info()) -> {ok, binary()} | pass.
convert(Path, #media_info{type = T, width = W, height = H}) ->
if W * H >= 25000000 ->
?DEBUG("The image ~s is more than 25 Mpix", [Path]),
pass;
W =< 300, H =< 300 ->
{ok, Path};
T == <<"gif">>; T == <<"jpeg">>; T == <<"png">>; T == <<"webp">> ->
Dir = filename:dirname(Path),
FileName = <<(randoms:get_string())/binary, $., T/binary>>,
OutPath = filename:join(Dir, FileName),
Cmd = io_lib:format("convert -resize 300 ~s ~s", [Path, OutPath]),
case os:cmd(Cmd) of
"" ->
{ok, OutPath};
Err ->
?ERROR_MSG("Failed to convert ~s to ~s: ~s",
[Path, OutPath, string:strip(Err, right, $\n)]),
pass
end;
true ->
?DEBUG("Won't call 'convert' for unknown type ~s", [T]),
pass
end.
-spec thumb_el(binary(), binary()) -> xmlel().
thumb_el(Path, URI) ->
ContentType = guess_content_type(Path),
case identify(Path) of
{ok, #media_info{height = H, width = W}} ->
#xmlel{name = <<"thumbnail">>,
attrs = [{<<"xmlns">>, ?NS_THUMBS_1},
{<<"media-type">>, ContentType},
{<<"uri">>, URI},
{<<"height">>, jlib:integer_to_binary(H)},
{<<"width">>, jlib:integer_to_binary(W)}]};
pass ->
#xmlel{name = <<"thumbnail">>,
attrs = [{<<"xmlns">>, ?NS_THUMBS_1},
{<<"uri">>, URI},
{<<"media-type">>, ContentType}]}
end.
%%--------------------------------------------------------------------
%% Remove user.
%%--------------------------------------------------------------------
@ -811,7 +965,7 @@ remove_user(User, Server) ->
?DEBUG("Found no HTTP upload directory of ~s@~s", [User, Server]);
{error, Error} ->
?ERROR_MSG("Cannot remove HTTP upload directory of ~s@~s: ~p",
[User, Server, Error])
[User, Server, ?FORMAT(Error)])
end,
ok.
@ -833,7 +987,7 @@ del_tree(Dir) ->
ok = file:del_dir(Dir)
catch
_:{badmatch, {error, Error}} ->
{error, ?FORMAT(Error)};
{error, Error};
_:Error ->
{error, Error}
end.

32
mod_http_upload/src/mod_http_upload_quota.erl
View File

@ -75,8 +75,8 @@ start(ServerHost, Opts) ->
stop(ServerHost) ->
Proc = mod_http_upload:get_proc_name(ServerHost, ?PROCNAME),
ok = supervisor:terminate_child(ejabberd_sup, Proc),
ok = supervisor:delete_child(ejabberd_sup, Proc).
supervisor:terminate_child(ejabberd_sup, Proc),
supervisor:delete_child(ejabberd_sup, Proc).
-spec mod_opt_type(atom()) -> fun((term()) -> term()) | [atom()].
@ -164,7 +164,7 @@ handle_cast({handle_slot_request, #jid{user = U, server = S} = JID, Path, Size},
{0, 0} ->
?DEBUG("No quota specified for ~s",
[jlib:jid_to_string(JID)]),
Size;
undefined;
{0, _} ->
?WARNING_MSG("No hard quota specified for ~s",
[jlib:jid_to_string(JID)]),
@ -183,7 +183,12 @@ handle_cast({handle_slot_request, #jid{user = U, server = S} = JID, Path, Size},
[jlib:jid_to_string(JID)]),
enforce_quota(Path, Size, OldSize, SoftQuota, HardQuota)
end,
{noreply, State#state{disk_usage = dict:store({U, S}, NewSize, DiskUsage)}};
NewDiskUsage = if is_integer(NewSize) ->
dict:store({U, S}, NewSize, DiskUsage);
true ->
DiskUsage
end,
{noreply, State#state{disk_usage = NewDiskUsage}};
handle_cast(Request, State) ->
?ERROR_MSG("Got unexpected request: ~p", [Request]),
{noreply, State}.
@ -199,11 +204,13 @@ handle_info(sweep, #state{server_host = ServerHost,
{ok, Entries} ->
BackThen = secs_since_epoch() - (MaxDays * 86400),
DocRootS = binary_to_list(DocRoot),
PathNames = lists:map(fun(Entry) -> DocRootS ++ "/" ++ Entry end,
Entries),
PathNames = lists:map(fun(Entry) ->
DocRootS ++ "/" ++ Entry
end, Entries),
UserDirs = lists:filter(fun filelib:is_dir/1, PathNames),
lists:foreach(fun(UserDir) -> delete_old_files(UserDir, BackThen) end,
UserDirs);
lists:foreach(fun(UserDir) ->
delete_old_files(UserDir, BackThen)
end, UserDirs);
{error, Error} ->
?ERROR_MSG("Cannot open document root ~s: ~s",
[DocRoot, ?FORMAT(Error)])
@ -276,9 +283,9 @@ enforce_quota(UserDir, SlotSize, _OldSize, MinSize, MaxSize) ->
-spec delete_old_files(file:filename_all(), integer()) -> ok.
delete_old_files(UserDir, Timestamp) ->
delete_old_files(UserDir, CutOff) ->
FileInfo = gather_file_info(UserDir),
case [Path || {Path, _Size, Time} <- FileInfo, Time < Timestamp] of
case [Path || {Path, _Size, Time} <- FileInfo, Time < CutOff] of
[] ->
ok;
OldFiles ->
@ -296,7 +303,8 @@ gather_file_info(Dir) ->
{ok, Entries} ->
lists:foldl(fun(Entry, Acc) ->
Path = Dir ++ "/" ++ Entry,
case file:read_file_info(Path, [{time, posix}]) of
case file:read_file_info(Path,
[{time, posix}]) of
{ok, #file_info{type = directory}} ->
gather_file_info(Path) ++ Acc;
{ok, #file_info{type = regular,
@ -332,7 +340,7 @@ del_file_and_dir(File) ->
ok ->
?DEBUG("Removed ~s", [Dir]);
{error, Error} ->
?INFO_MSG("Cannot remove ~s: ~s", [Dir, ?FORMAT(Error)])
?DEBUG("Cannot remove ~s: ~s", [Dir, ?FORMAT(Error)])
end;
{error, Error} ->
?WARNING_MSG("Cannot remove ~s: ~s", [File, ?FORMAT(Error)])