New script make-deps.sh to get deps, and bitarray fixed rebar.config (#296)
This commit is contained in:
Badlop
parent
dd4c264b33
commit
f46182b238
|
|
@ -19,15 +19,9 @@ This is how I got it to work... YMMV.
|
||||||
1. Make sure ejabberd is running
|
1. Make sure ejabberd is running
|
||||||
2. Get the updated ejabberd-contrib sources:
|
2. Get the updated ejabberd-contrib sources:
|
||||||
ejabberdctl modules_update_specs
|
ejabberdctl modules_update_specs
|
||||||
3. Then get manually the dependencies, as the scripts can't do it:
|
3. Execute this script to get and build the dependencies:
|
||||||
cd ~/.ejabberd-modules/sources/ejabberd-contrib/mod_pottymouth/deps/etbloom/
|
~/.ejabberd-modules/sources/ejabberd-contrib/mod_pottymouth/make-deps.sh
|
||||||
chmod +x rebar
|
4. When you install the module, its dependencies will be installed as well:
|
||||||
./rebar get-deps
|
|
||||||
./rebar compile
|
|
||||||
cp -R deps/bitarray/ebin ../../
|
|
||||||
cp -R deps/bitarray/priv ../../
|
|
||||||
cp -R deps/proper/ebin ../../
|
|
||||||
4. Finally, when you install the module, its dependencies will be installed as well:
|
|
||||||
ejabberdctl module_install mod_pottymouth
|
ejabberdctl module_install mod_pottymouth
|
||||||
|
|
||||||
Configuration file is ~/.ejabberd-modules/mod_pottymouth/conf/mod_pottymouth.yml
|
Configuration file is ~/.ejabberd-modules/mod_pottymouth/conf/mod_pottymouth.yml
|
||||||
|
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
workspace.xml
|
|
||||||
out
|
|
||||||
|
|
@ -1,286 +0,0 @@
|
||||||
ERLANG PUBLIC LICENSE
|
|
||||||
Version 1.1
|
|
||||||
|
|
||||||
1. Definitions.
|
|
||||||
|
|
||||||
1.1. ``Contributor'' means each entity that creates or contributes to
|
|
||||||
the creation of Modifications.
|
|
||||||
|
|
||||||
1.2. ``Contributor Version'' means the combination of the Original
|
|
||||||
Code, prior Modifications used by a Contributor, and the Modifications
|
|
||||||
made by that particular Contributor.
|
|
||||||
|
|
||||||
1.3. ``Covered Code'' means the Original Code or Modifications or the
|
|
||||||
combination of the Original Code and Modifications, in each case
|
|
||||||
including portions thereof.
|
|
||||||
|
|
||||||
1.4. ``Electronic Distribution Mechanism'' means a mechanism generally
|
|
||||||
accepted in the software development community for the electronic
|
|
||||||
transfer of data.
|
|
||||||
|
|
||||||
1.5. ``Executable'' means Covered Code in any form other than Source
|
|
||||||
Code.
|
|
||||||
|
|
||||||
1.6. ``Initial Developer'' means the individual or entity identified
|
|
||||||
as the Initial Developer in the Source Code notice required by Exhibit
|
|
||||||
A.
|
|
||||||
|
|
||||||
1.7. ``Larger Work'' means a work which combines Covered Code or
|
|
||||||
portions thereof with code not governed by the terms of this License.
|
|
||||||
|
|
||||||
1.8. ``License'' means this document.
|
|
||||||
|
|
||||||
1.9. ``Modifications'' means any addition to or deletion from the
|
|
||||||
substance or structure of either the Original Code or any previous
|
|
||||||
Modifications. When Covered Code is released as a series of files, a
|
|
||||||
Modification is:
|
|
||||||
|
|
||||||
A. Any addition to or deletion from the contents of a file containing
|
|
||||||
Original Code or previous Modifications.
|
|
||||||
|
|
||||||
B. Any new file that contains any part of the Original Code or
|
|
||||||
previous Modifications.
|
|
||||||
|
|
||||||
1.10. ``Original Code'' means Source Code of computer software code
|
|
||||||
which is described in the Source Code notice required by Exhibit A as
|
|
||||||
Original Code, and which, at the time of its release under this
|
|
||||||
License is not already Covered Code governed by this License.
|
|
||||||
|
|
||||||
1.11. ``Source Code'' means the preferred form of the Covered Code for
|
|
||||||
making modifications to it, including all modules it contains, plus
|
|
||||||
any associated interface definition files, scripts used to control
|
|
||||||
compilation and installation of an Executable, or a list of source
|
|
||||||
code differential comparisons against either the Original Code or
|
|
||||||
another well known, available Covered Code of the Contributor's
|
|
||||||
choice. The Source Code can be in a compressed or archival form,
|
|
||||||
provided the appropriate decompression or de-archiving software is
|
|
||||||
widely available for no charge.
|
|
||||||
|
|
||||||
1.12. ``You'' means an individual or a legal entity exercising rights
|
|
||||||
under, and complying with all of the terms of, this License. For legal
|
|
||||||
entities,``You'' includes any entity which controls, is controlled by,
|
|
||||||
or is under common control with You. For purposes of this definition,
|
|
||||||
``control'' means (a) the power, direct or indirect, to cause the
|
|
||||||
direction or management of such entity, whether by contract or
|
|
||||||
otherwise, or (b) ownership of fifty percent (50%) or more of the
|
|
||||||
outstanding shares or beneficial ownership of such entity.
|
|
||||||
|
|
||||||
2. Source Code License.
|
|
||||||
|
|
||||||
2.1. The Initial Developer Grant.
|
|
||||||
The Initial Developer hereby grants You a world-wide, royalty-free,
|
|
||||||
non-exclusive license, subject to third party intellectual property
|
|
||||||
claims:
|
|
||||||
|
|
||||||
(a) to use, reproduce, modify, display, perform, sublicense and
|
|
||||||
distribute the Original Code (or portions thereof) with or without
|
|
||||||
Modifications, or as part of a Larger Work; and
|
|
||||||
|
|
||||||
(b) under patents now or hereafter owned or controlled by Initial
|
|
||||||
Developer, to make, have made, use and sell (``Utilize'') the
|
|
||||||
Original Code (or portions thereof), but solely to the extent that
|
|
||||||
any such patent is reasonably necessary to enable You to Utilize
|
|
||||||
the Original Code (or portions thereof) and not to any greater
|
|
||||||
extent that may be necessary to Utilize further Modifications or
|
|
||||||
combinations.
|
|
||||||
|
|
||||||
2.2. Contributor Grant.
|
|
||||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
|
||||||
non-exclusive license, subject to third party intellectual property
|
|
||||||
claims:
|
|
||||||
|
|
||||||
(a) to use, reproduce, modify, display, perform, sublicense and
|
|
||||||
distribute the Modifications created by such Contributor (or
|
|
||||||
portions thereof) either on an unmodified basis, with other
|
|
||||||
Modifications, as Covered Code or as part of a Larger Work; and
|
|
||||||
|
|
||||||
(b) under patents now or hereafter owned or controlled by Contributor,
|
|
||||||
to Utilize the Contributor Version (or portions thereof), but
|
|
||||||
solely to the extent that any such patent is reasonably necessary
|
|
||||||
to enable You to Utilize the Contributor Version (or portions
|
|
||||||
thereof), and not to any greater extent that may be necessary to
|
|
||||||
Utilize further Modifications or combinations.
|
|
||||||
|
|
||||||
3. Distribution Obligations.
|
|
||||||
|
|
||||||
3.1. Application of License.
|
|
||||||
The Modifications which You contribute are governed by the terms of
|
|
||||||
this License, including without limitation Section 2.2. The Source
|
|
||||||
Code version of Covered Code may be distributed only under the terms
|
|
||||||
of this License, and You must include a copy of this License with
|
|
||||||
every copy of the Source Code You distribute. You may not offer or
|
|
||||||
impose any terms on any Source Code version that alters or restricts
|
|
||||||
the applicable version of this License or the recipients' rights
|
|
||||||
hereunder. However, You may include an additional document offering
|
|
||||||
the additional rights described in Section 3.5.
|
|
||||||
|
|
||||||
3.2. Availability of Source Code.
|
|
||||||
Any Modification which You contribute must be made available in Source
|
|
||||||
Code form under the terms of this License either on the same media as
|
|
||||||
an Executable version or via an accepted Electronic Distribution
|
|
||||||
Mechanism to anyone to whom you made an Executable version available;
|
|
||||||
and if made available via Electronic Distribution Mechanism, must
|
|
||||||
remain available for at least twelve (12) months after the date it
|
|
||||||
initially became available, or at least six (6) months after a
|
|
||||||
subsequent version of that particular Modification has been made
|
|
||||||
available to such recipients. You are responsible for ensuring that
|
|
||||||
the Source Code version remains available even if the Electronic
|
|
||||||
Distribution Mechanism is maintained by a third party.
|
|
||||||
|
|
||||||
3.3. Description of Modifications.
|
|
||||||
You must cause all Covered Code to which you contribute to contain a
|
|
||||||
file documenting the changes You made to create that Covered Code and
|
|
||||||
the date of any change. You must include a prominent statement that
|
|
||||||
the Modification is derived, directly or indirectly, from Original
|
|
||||||
Code provided by the Initial Developer and including the name of the
|
|
||||||
Initial Developer in (a) the Source Code, and (b) in any notice in an
|
|
||||||
Executable version or related documentation in which You describe the
|
|
||||||
origin or ownership of the Covered Code.
|
|
||||||
|
|
||||||
3.4. Intellectual Property Matters
|
|
||||||
|
|
||||||
(a) Third Party Claims.
|
|
||||||
If You have knowledge that a party claims an intellectual property
|
|
||||||
right in particular functionality or code (or its utilization
|
|
||||||
under this License), you must include a text file with the source
|
|
||||||
code distribution titled ``LEGAL'' which describes the claim and
|
|
||||||
the party making the claim in sufficient detail that a recipient
|
|
||||||
will know whom to contact. If you obtain such knowledge after You
|
|
||||||
make Your Modification available as described in Section 3.2, You
|
|
||||||
shall promptly modify the LEGAL file in all copies You make
|
|
||||||
available thereafter and shall take other steps (such as notifying
|
|
||||||
appropriate mailing lists or newsgroups) reasonably calculated to
|
|
||||||
inform those who received the Covered Code that new knowledge has
|
|
||||||
been obtained.
|
|
||||||
|
|
||||||
(b) Contributor APIs.
|
|
||||||
If Your Modification is an application programming interface and
|
|
||||||
You own or control patents which are reasonably necessary to
|
|
||||||
implement that API, you must also include this information in the
|
|
||||||
LEGAL file.
|
|
||||||
|
|
||||||
3.5. Required Notices.
|
|
||||||
You must duplicate the notice in Exhibit A in each file of the Source
|
|
||||||
Code, and this License in any documentation for the Source Code, where
|
|
||||||
You describe recipients' rights relating to Covered Code. If You
|
|
||||||
created one or more Modification(s), You may add your name as a
|
|
||||||
Contributor to the notice described in Exhibit A. If it is not
|
|
||||||
possible to put such notice in a particular Source Code file due to
|
|
||||||
its structure, then you must include such notice in a location (such
|
|
||||||
as a relevant directory file) where a user would be likely to look for
|
|
||||||
such a notice. You may choose to offer, and to charge a fee for,
|
|
||||||
warranty, support, indemnity or liability obligations to one or more
|
|
||||||
recipients of Covered Code. However, You may do so only on Your own
|
|
||||||
behalf, and not on behalf of the Initial Developer or any
|
|
||||||
Contributor. You must make it absolutely clear than any such warranty,
|
|
||||||
support, indemnity or liability obligation is offered by You alone,
|
|
||||||
and You hereby agree to indemnify the Initial Developer and every
|
|
||||||
Contributor for any liability incurred by the Initial Developer or
|
|
||||||
such Contributor as a result of warranty, support, indemnity or
|
|
||||||
liability terms You offer.
|
|
||||||
|
|
||||||
3.6. Distribution of Executable Versions.
|
|
||||||
You may distribute Covered Code in Executable form only if the
|
|
||||||
requirements of Section 3.1-3.5 have been met for that Covered Code,
|
|
||||||
and if You include a notice stating that the Source Code version of
|
|
||||||
the Covered Code is available under the terms of this License,
|
|
||||||
including a description of how and where You have fulfilled the
|
|
||||||
obligations of Section 3.2. The notice must be conspicuously included
|
|
||||||
in any notice in an Executable version, related documentation or
|
|
||||||
collateral in which You describe recipients' rights relating to the
|
|
||||||
Covered Code. You may distribute the Executable version of Covered
|
|
||||||
Code under a license of Your choice, which may contain terms different
|
|
||||||
from this License, provided that You are in compliance with the terms
|
|
||||||
of this License and that the license for the Executable version does
|
|
||||||
not attempt to limit or alter the recipient's rights in the Source
|
|
||||||
Code version from the rights set forth in this License. If You
|
|
||||||
distribute the Executable version under a different license You must
|
|
||||||
make it absolutely clear that any terms which differ from this License
|
|
||||||
are offered by You alone, not by the Initial Developer or any
|
|
||||||
Contributor. You hereby agree to indemnify the Initial Developer and
|
|
||||||
every Contributor for any liability incurred by the Initial Developer
|
|
||||||
or such Contributor as a result of any such terms You offer.
|
|
||||||
|
|
||||||
3.7. Larger Works.
|
|
||||||
You may create a Larger Work by combining Covered Code with other code
|
|
||||||
not governed by the terms of this License and distribute the Larger
|
|
||||||
Work as a single product. In such a case, You must make sure the
|
|
||||||
requirements of this License are fulfilled for the Covered Code.
|
|
||||||
|
|
||||||
4. Inability to Comply Due to Statute or Regulation.
|
|
||||||
If it is impossible for You to comply with any of the terms of this
|
|
||||||
License with respect to some or all of the Covered Code due to statute
|
|
||||||
or regulation then You must: (a) comply with the terms of this License
|
|
||||||
to the maximum extent possible; and (b) describe the limitations and
|
|
||||||
the code they affect. Such description must be included in the LEGAL
|
|
||||||
file described in Section 3.4 and must be included with all
|
|
||||||
distributions of the Source Code. Except to the extent prohibited by
|
|
||||||
statute or regulation, such description must be sufficiently detailed
|
|
||||||
for a recipient of ordinary skill to be able to understand it.
|
|
||||||
|
|
||||||
5. Application of this License.
|
|
||||||
|
|
||||||
This License applies to code to which the Initial Developer has
|
|
||||||
attached the notice in Exhibit A, and to related Covered Code.
|
|
||||||
|
|
||||||
6. CONNECTION TO MOZILLA PUBLIC LICENSE
|
|
||||||
|
|
||||||
This Erlang License is a derivative work of the Mozilla Public
|
|
||||||
License, Version 1.0. It contains terms which differ from the Mozilla
|
|
||||||
Public License, Version 1.0.
|
|
||||||
|
|
||||||
7. DISCLAIMER OF WARRANTY.
|
|
||||||
|
|
||||||
COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN ``AS IS'' BASIS,
|
|
||||||
WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
|
|
||||||
WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
|
|
||||||
DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR
|
|
||||||
NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF
|
|
||||||
THE COVERED CODE IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE
|
|
||||||
IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER
|
|
||||||
CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR
|
|
||||||
CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART
|
|
||||||
OF THIS LICENSE. NO USE OF ANY COVERED CODE IS AUTHORIZED HEREUNDER
|
|
||||||
EXCEPT UNDER THIS DISCLAIMER.
|
|
||||||
|
|
||||||
8. TERMINATION.
|
|
||||||
This License and the rights granted hereunder will terminate
|
|
||||||
automatically if You fail to comply with terms herein and fail to cure
|
|
||||||
such breach within 30 days of becoming aware of the breach. All
|
|
||||||
sublicenses to the Covered Code which are properly granted shall
|
|
||||||
survive any termination of this License. Provisions which, by their
|
|
||||||
nature, must remain in effect beyond the termination of this License
|
|
||||||
shall survive.
|
|
||||||
|
|
||||||
9. DISCLAIMER OF LIABILITY
|
|
||||||
Any utilization of Covered Code shall not cause the Initial Developer
|
|
||||||
or any Contributor to be liable for any damages (neither direct nor
|
|
||||||
indirect).
|
|
||||||
|
|
||||||
10. MISCELLANEOUS
|
|
||||||
This License represents the complete agreement concerning the subject
|
|
||||||
matter hereof. If any provision is held to be unenforceable, such
|
|
||||||
provision shall be reformed only to the extent necessary to make it
|
|
||||||
enforceable. This License shall be construed by and in accordance with
|
|
||||||
the substantive laws of Sweden. Any dispute, controversy or claim
|
|
||||||
arising out of or relating to this License, or the breach, termination
|
|
||||||
or invalidity thereof, shall be subject to the exclusive jurisdiction
|
|
||||||
of Swedish courts, with the Stockholm City Court as the first
|
|
||||||
instance.
|
|
||||||
|
|
||||||
EXHIBIT A.
|
|
||||||
|
|
||||||
``The contents of this file are subject to the Erlang Public License,
|
|
||||||
Version 1.1, (the "License"); you may not use this file except in
|
|
||||||
compliance with the License. You should have received a copy of the
|
|
||||||
Erlang Public License along with this software. If not, it can be
|
|
||||||
retrieved via the world wide web at http://www.erlang.org/.
|
|
||||||
|
|
||||||
Software distributed under the License is distributed on an "AS IS"
|
|
||||||
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
|
|
||||||
the License for the specific language governing rights and limitations
|
|
||||||
under the License.
|
|
||||||
|
|
||||||
The Initial Developer of the Original Code is Ericsson Utvecklings AB.
|
|
||||||
Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
|
|
||||||
AB. All Rights Reserved.''
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
Erlang Bloom Filter
|
|
||||||
=======
|
|
||||||
|
|
||||||
Based on Scalable Bloom Filters Paulo Sérgio Almeida, Carlos Baquero, Nuno Preguiça, David Hutchison
|
|
||||||
Information Processing Letters Volume 101, Issue 6, 31 March 2007, Pages 255-261
|
|
||||||
|
|
||||||
Provides scalable bloom filters that can grow indefinitely while
|
|
||||||
ensuring a desired maximum false positive probability. Also provides
|
|
||||||
standard partitioned bloom filters with a maximum capacity. Bit arrays
|
|
||||||
are dimensioned as a power of 2 to enable reusing hash values across
|
|
||||||
filters through bit operations. Double hashing is used (no need for
|
|
||||||
enhanced double hashing for partitioned bloom filters).
|
|
||||||
|
|
||||||
Binary file not shown.
|
|
@ -1,5 +0,0 @@
|
||||||
{erl_opts, [warnings_as_errors, debug_info]}.
|
|
||||||
{deps, [
|
|
||||||
{bitarray, "0.1",
|
|
||||||
{git, "https://github.com/madglory/bitarray", {branch, "master"}}}
|
|
||||||
]}.
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
%% The contents of this file are subject to the Erlang Public License,
|
|
||||||
%% Version 1.1, (the "License"); you may not use this file except in
|
|
||||||
%% compliance with the License. You should have received a copy of the
|
|
||||||
%% Erlang Public License along with this software. If not, it can be
|
|
||||||
%% retrieved via the world wide web at http://www.erlang.org/.
|
|
||||||
%%
|
|
||||||
%% Software distributed under the License is distributed on an "AS IS"
|
|
||||||
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
|
|
||||||
%% the License for the specific language governing rights and limitations
|
|
||||||
%% under the License.
|
|
||||||
%%
|
|
||||||
%% {application, etbloom, [
|
|
||||||
%% {description, "Erlang Bloom"},
|
|
||||||
%% {registered, []},
|
|
||||||
%% {applications, [
|
|
||||||
%% kernel,
|
|
||||||
%% stdlib
|
|
||||||
%% ]},
|
|
||||||
%% {env, []}
|
|
||||||
%% ]}.
|
|
||||||
{application, etbloom,
|
|
||||||
[{description, "Erlang Bloom"},
|
|
||||||
{vsn, "1.1.0"},
|
|
||||||
{registered, []},
|
|
||||||
{mod, { etbloom_app, []}},
|
|
||||||
{applications,
|
|
||||||
[kernel,
|
|
||||||
stdlib,
|
|
||||||
bitarray
|
|
||||||
]},
|
|
||||||
{env,[]},
|
|
||||||
{modules, []},
|
|
||||||
|
|
||||||
{maintainers, []},
|
|
||||||
{licenses, []},
|
|
||||||
{links, []}
|
|
||||||
]}.
|
|
||||||
|
|
@ -1,174 +0,0 @@
|
||||||
%% The contents of this file are subject to the Erlang Public License,
|
|
||||||
%% Version 1.1, (the "License"); you may not use this file except in
|
|
||||||
%% compliance with the License. You should have received a copy of the
|
|
||||||
%% Erlang Public License along with this software. If not, it can be
|
|
||||||
%% retrieved via the world wide web at http://www.erlang.org/.
|
|
||||||
%%
|
|
||||||
%% Software distributed under the License is distributed on an "AS IS"
|
|
||||||
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
|
|
||||||
%% the License for the specific language governing rights and limitations
|
|
||||||
%% under the License.
|
|
||||||
%%
|
|
||||||
-module(etbloom).
|
|
||||||
-author("Paulo Sergio Almeida <psa@di.uminho.pt>").
|
|
||||||
-export([sbf/1, sbf/2, sbf/3, sbf/4,
|
|
||||||
bloom/1, bloom/2,
|
|
||||||
member/2, add/2,
|
|
||||||
size/1, capacity/1]).
|
|
||||||
-export([is_element/2, add_element/2]). % alternative names
|
|
||||||
-import(math, [log/1, pow/2]).
|
|
||||||
|
|
||||||
is_element(E, B) -> member(E, B).
|
|
||||||
add_element(E, B) -> add(E, B).
|
|
||||||
|
|
||||||
|
|
||||||
%% Based on
|
|
||||||
%% Scalable Bloom Filters
|
|
||||||
%% Paulo Sérgio Almeida, Carlos Baquero, Nuno Preguiça, David Hutchison
|
|
||||||
%% Information Processing Letters
|
|
||||||
%% Volume 101, Issue 6, 31 March 2007, Pages 255-261
|
|
||||||
%%
|
|
||||||
%% Provides scalable bloom filters that can grow indefinitely while
|
|
||||||
%% ensuring a desired maximum false positive probability. Also provides
|
|
||||||
%% standard partitioned bloom filters with a maximum capacity. Bit arrays
|
|
||||||
%% are dimensioned as a power of 2 to enable reusing hash values across
|
|
||||||
%% filters through bit operations. Double hashing is used (no need for
|
|
||||||
%% enhanced double hashing for partitioned bloom filters).
|
|
||||||
%%
|
|
||||||
|
|
||||||
-record(bloom, {
|
|
||||||
e, % error probability
|
|
||||||
n, % maximum number of elements
|
|
||||||
mb, % 2^mb = m, the size of each slice (bitvector)
|
|
||||||
size, % number of elements
|
|
||||||
a % list of bitvectors
|
|
||||||
}).
|
|
||||||
|
|
||||||
-record(sbf, {
|
|
||||||
e, % error probability
|
|
||||||
r, % error probability ratio
|
|
||||||
s, % log 2 of growth ratio
|
|
||||||
size, % number of elements
|
|
||||||
b % list of plain bloom filters
|
|
||||||
}).
|
|
||||||
|
|
||||||
%% Constructors for (fixed capacity) bloom filters
|
|
||||||
%%
|
|
||||||
%% N - capacity
|
|
||||||
%% E - error probability
|
|
||||||
|
|
||||||
bloom(L) when is_list(L) -> lists:foldl(fun(X, Bloom) -> add(X, Bloom) end, bloom(length(L)), L);
|
|
||||||
bloom(N) when is_integer(N) -> bloom(N, 0.001).
|
|
||||||
bloom(N, E) when is_number(N), N > 0,
|
|
||||||
is_float(E), E > 0, E < 1,
|
|
||||||
N >= 4 / E -> % rule of thumb; due to double hashing
|
|
||||||
bloom(size, N, E).
|
|
||||||
|
|
||||||
bloom(Mode, Dim, E) ->
|
|
||||||
K = 1 + trunc(log2(1 / E)),
|
|
||||||
P = pow(E, 1 / K),
|
|
||||||
case Mode of
|
|
||||||
size -> Mb = 1 + trunc(-log2(1 - pow(1 - P, 1 / Dim)));
|
|
||||||
bits -> Mb = Dim
|
|
||||||
end,
|
|
||||||
M = 1 bsl Mb,
|
|
||||||
N = trunc(log(1 - P) / log(1 - 1 / M)),
|
|
||||||
#bloom{e = E, n = N, mb = Mb, size = 0,
|
|
||||||
a = [bitarray:new(1 bsl Mb, false) || _ <- lists:seq(1, K)]}.
|
|
||||||
|
|
||||||
log2(X) -> log(X) / log(2).
|
|
||||||
|
|
||||||
%% Constructors for scalable bloom filters
|
|
||||||
%%
|
|
||||||
%% N - initial capacity before expanding
|
|
||||||
%% E - error probability
|
|
||||||
%% S - growth ratio when full (log 2) can be 1, 2 or 3
|
|
||||||
%% R - tightening ratio of error probability
|
|
||||||
|
|
||||||
sbf(N) -> sbf(N, 0.001).
|
|
||||||
sbf(N, E) -> sbf(N, E, 1).
|
|
||||||
sbf(N, E, 1) -> sbf(N, E, 1, 0.85);
|
|
||||||
sbf(N, E, 2) -> sbf(N, E, 2, 0.75);
|
|
||||||
sbf(N, E, 3) -> sbf(N, E, 3, 0.65).
|
|
||||||
sbf(N, E, S, R) when is_number(N), N > 0,
|
|
||||||
is_float(E), E > 0, E < 1,
|
|
||||||
is_integer(S), S > 0, S < 4,
|
|
||||||
is_float(R), R > 0, R < 1,
|
|
||||||
N >= 4 / (E * (1 - R)) -> % rule of thumb; due to double hashing
|
|
||||||
#sbf{e = E, s = S, r = R, size = 0, b = [bloom(N, E * (1 - R))]}.
|
|
||||||
|
|
||||||
%% Returns number of elements
|
|
||||||
%%
|
|
||||||
size(#bloom{size = Size}) -> Size;
|
|
||||||
size(#sbf{size = Size}) -> Size.
|
|
||||||
|
|
||||||
%% Returns capacity
|
|
||||||
%%
|
|
||||||
capacity(#bloom{n = N}) -> N;
|
|
||||||
capacity(#sbf{}) -> infinity.
|
|
||||||
|
|
||||||
%% Test for membership
|
|
||||||
%%
|
|
||||||
member(Elem, #bloom{mb = Mb} = B) ->
|
|
||||||
Hashes = make_hashes(Mb, Elem),
|
|
||||||
hash_member(Hashes, B);
|
|
||||||
member(Elem, #sbf{b = [H | _]} = Sbf) ->
|
|
||||||
Hashes = make_hashes(H#bloom.mb, Elem),
|
|
||||||
hash_member(Hashes, Sbf).
|
|
||||||
|
|
||||||
hash_member(Hashes, #bloom{mb = Mb, a = A}) ->
|
|
||||||
Mask = 1 bsl Mb - 1,
|
|
||||||
{I1, I0} = make_indexes(Mask, Hashes),
|
|
||||||
all_set(Mask, I1, I0, A);
|
|
||||||
hash_member(Hashes, #sbf{b = B}) ->
|
|
||||||
lists:any(fun(X) -> hash_member(Hashes, X) end, B).
|
|
||||||
|
|
||||||
make_hashes(Mb, E) when Mb =< 16 ->
|
|
||||||
erlang:phash2({E}, 1 bsl 32);
|
|
||||||
make_hashes(Mb, E) when Mb =< 32 ->
|
|
||||||
{erlang:phash2({E}, 1 bsl 32), erlang:phash2([E], 1 bsl 32)}.
|
|
||||||
|
|
||||||
make_indexes(Mask, {H0, H1}) when Mask > 1 bsl 16 -> masked_pair(Mask, H0, H1);
|
|
||||||
make_indexes(Mask, {H0, _}) -> make_indexes(Mask, H0);
|
|
||||||
make_indexes(Mask, H0) -> masked_pair(Mask, H0 bsr 16, H0).
|
|
||||||
|
|
||||||
masked_pair(Mask, X, Y) -> {X band Mask, Y band Mask}.
|
|
||||||
|
|
||||||
all_set(_Mask, _I1, _I, []) -> true;
|
|
||||||
all_set(Mask, I1, I, [H | T]) ->
|
|
||||||
case bitarray:sub(H, I) of
|
|
||||||
true -> all_set(Mask, I1, (I + I1) band Mask, T);
|
|
||||||
false -> false
|
|
||||||
end.
|
|
||||||
|
|
||||||
%% Adds element to set
|
|
||||||
%%
|
|
||||||
add(Elem, #bloom{mb = Mb} = B) ->
|
|
||||||
Hashes = make_hashes(Mb, Elem),
|
|
||||||
hash_add(Hashes, B);
|
|
||||||
add(Elem, #sbf{size = Size, r = R, s = S, b = [H | T] = Bs} = Sbf) ->
|
|
||||||
#bloom{mb = Mb, e = E, n = N, size = HSize} = H,
|
|
||||||
Hashes = make_hashes(Mb, Elem),
|
|
||||||
case hash_member(Hashes, Sbf) of
|
|
||||||
true -> Sbf;
|
|
||||||
false ->
|
|
||||||
case HSize < N of
|
|
||||||
true -> Sbf#sbf{size = Size + 1, b = [hash_add(Hashes, H) | T]};
|
|
||||||
false ->
|
|
||||||
B = add(Elem, bloom(bits, Mb + S, E * R)),
|
|
||||||
Sbf#sbf{size = Size + 1, b = [B | Bs]}
|
|
||||||
end
|
|
||||||
end.
|
|
||||||
|
|
||||||
hash_add(Hashes, #bloom{mb = Mb, a = A, size = Size} = B) ->
|
|
||||||
Mask = 1 bsl Mb - 1,
|
|
||||||
{I1, I0} = make_indexes(Mask, Hashes),
|
|
||||||
case all_set(Mask, I1, I0, A) of
|
|
||||||
true -> B;
|
|
||||||
false -> B#bloom{size = Size + 1, a = set_bits(Mask, I1, I0, A, [])}
|
|
||||||
end.
|
|
||||||
|
|
||||||
set_bits(_Mask, _I1, _I, [], Acc) -> lists:reverse(Acc);
|
|
||||||
set_bits(Mask, I1, I, [H | T], Acc) ->
|
|
||||||
set_bits(Mask, I1, (I + I1) band Mask, T, [bitarray:update(H, I, true) | Acc]).
|
|
||||||
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
%% The contents of this file are subject to the Erlang Public License,
|
|
||||||
%% Version 1.1, (the "License"); you may not use this file except in
|
|
||||||
%% compliance with the License. You should have received a copy of the
|
|
||||||
%% Erlang Public License along with this software. If not, it can be
|
|
||||||
%% retrieved via the world wide web at http://www.erlang.org/.
|
|
||||||
%%
|
|
||||||
%% Software distributed under the License is distributed on an "AS IS"
|
|
||||||
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
|
|
||||||
%% the License for the specific language governing rights and limitations
|
|
||||||
%% under the License.
|
|
||||||
%%
|
|
||||||
-module(etbloom_tests).
|
|
||||||
-author("volodymyr.kyrychenko@strikead.com").
|
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
|
||||||
|
|
||||||
bloom_test() ->
|
|
||||||
Values = [{xxx, binary_to_atom(base64:encode(crypto:strong_rand_bytes(10)), utf8)} || _ <- lists:seq(1, 5000)],
|
|
||||||
Bloom = etbloom:bloom(Values),
|
|
||||||
?assert(lists:all(fun(X) -> etbloom:member(X, Bloom) end, Values)),
|
|
||||||
?assertNot(etbloom:member(wtf, Bloom)).
|
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
PWD=`pwd`
|
||||||
|
|
||||||
|
cd ~/.ejabberd-modules/sources/ejabberd-contrib/mod_pottymouth/
|
||||||
|
mkdir deps
|
||||||
|
cd deps
|
||||||
|
git clone https://github.com/madglory/etbloom.git
|
||||||
|
cd etbloom
|
||||||
|
rm -rf deps/
|
||||||
|
./rebar get-deps
|
||||||
|
cp ../../rebar.config.bitarray deps/bitarray/rebar.config
|
||||||
|
./rebar compile
|
||||||
|
cp -R deps/bitarray/ebin ../../
|
||||||
|
cp -R deps/bitarray/priv ../../
|
||||||
|
cp -R deps/proper/ebin ../../
|
||||||
|
|
||||||
|
cd $PWD
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
{ct_extra_params, "-env ERL_LIBS deps/"}.
|
||||||
|
{port_env,
|
||||||
|
[ {"DRV_CFLAGS", "-fPIC -O2 $ERL_CFLAGS"},
|
||||||
|
{"ERL_LDFLAGS", " -L$ERL_EI_LIBDIR -lei"}
|
||||||
|
]}.
|
||||||
|
{port_specs, [{"priv/bitarray.so", ["c_src/bitarray_nif.c"]}]}.
|
||||||
|
|
||||||
|
|
||||||
|
{deps, [
|
||||||
|
%% For runtime
|
||||||
|
%% --
|
||||||
|
%% For tests
|
||||||
|
{proper, "1.2",
|
||||||
|
{git, "https://github.com/manopapad/proper.git", {tag, "v1.2"}}}]}.
|
||||||
Loading…
Reference in New Issue