Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F80327533
DetectDuplicates.java
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Sat, Aug 31, 04:24
Size
2 KB
Mime Type
text/x-c
Expires
Mon, Sep 2, 04:24 (2 d)
Engine
blob
Format
Raw Data
Handle
20300217
Attached To
R1473 warcbase
DetectDuplicates.java
View Options
package
org.warcbase.analysis
;
import
java.io.IOException
;
import
java.util.Arrays
;
import
org.apache.commons.cli.CommandLine
;
import
org.apache.commons.cli.CommandLineParser
;
import
org.apache.commons.cli.GnuParser
;
import
org.apache.commons.cli.HelpFormatter
;
import
org.apache.commons.cli.OptionBuilder
;
import
org.apache.commons.cli.Options
;
import
org.apache.commons.cli.ParseException
;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.hadoop.hbase.HBaseConfiguration
;
import
org.apache.hadoop.hbase.client.HTable
;
import
org.apache.hadoop.hbase.client.Result
;
import
org.apache.hadoop.hbase.client.ResultScanner
;
import
org.apache.hadoop.hbase.client.Scan
;
import
org.warcbase.ingest.IngestFiles
;
public
class
DetectDuplicates
{
private
static
final
String
NAME_OPTION
=
"name"
;
@SuppressWarnings
(
"static-access"
)
public
static
void
main
(
String
[]
args
)
throws
IOException
{
Options
options
=
new
Options
();
options
.
addOption
(
OptionBuilder
.
withArgName
(
"name"
).
hasArg
()
.
withDescription
(
"name of the archive"
).
create
(
NAME_OPTION
));
Configuration
hbaseConfig
=
HBaseConfiguration
.
create
();
CommandLine
cmdline
=
null
;
CommandLineParser
parser
=
new
GnuParser
();
try
{
cmdline
=
parser
.
parse
(
options
,
args
);
}
catch
(
ParseException
exp
)
{
System
.
err
.
println
(
"Error parsing command line: "
+
exp
.
getMessage
());
System
.
exit
(-
1
);
}
if
(!
cmdline
.
hasOption
(
NAME_OPTION
))
{
HelpFormatter
formatter
=
new
HelpFormatter
();
formatter
.
printHelp
(
IngestFiles
.
class
.
getCanonicalName
(),
options
);
System
.
exit
(-
1
);
}
String
name
=
cmdline
.
getOptionValue
(
NAME_OPTION
);
HTable
table
=
new
HTable
(
hbaseConfig
,
name
);
Scan
scan
=
new
Scan
();
ResultScanner
scanner
=
table
.
getScanner
(
scan
);
int
duplicates
=
0
;
long
duplicateSize
=
0
;
int
progress
=
0
;
for
(
Result
rr
=
scanner
.
next
();
rr
!=
null
;
rr
=
scanner
.
next
())
{
progress
++;
for
(
int
i
=
0
;
i
<
rr
.
raw
().
length
;
i
++)
for
(
int
j
=
i
+
1
;
j
<
rr
.
raw
().
length
;
j
++)
{
if
(
rr
.
raw
()[
i
].
getValue
().
length
!=
rr
.
raw
()[
j
].
getValue
().
length
)
{
continue
;
}
if
(
Arrays
.
equals
(
rr
.
raw
()[
i
].
getValue
(),
rr
.
raw
()[
j
].
getValue
()))
{
duplicates
++;
duplicateSize
+=
rr
.
raw
()[
i
].
getValue
().
length
;
}
}
if
(
progress
%
10000
==
0
)
{
System
.
out
.
println
(
"Done with "
+
progress
+
" rows. duplicates = "
+
duplicates
);
}
}
table
.
close
();
System
.
out
.
println
(
"Number of Duplicates: "
+
duplicates
);
System
.
out
.
println
(
"Total Duplicate size: "
+
duplicateSize
);
}
}
Event Timeline
Log In to Comment