Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F85283695
PrintAllUris.java
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Sat, Sep 28, 00:22
Size
3 KB
Mime Type
text/html
Expires
Mon, Sep 30, 00:22 (2 d)
Engine
blob
Format
Raw Data
Handle
21154939
Attached To
R1473 warcbase
PrintAllUris.java
View Options
package
org.warcbase.analysis
;
import
java.io.IOException
;
import
java.io.PrintWriter
;
import
org.apache.commons.cli.CommandLine
;
import
org.apache.commons.cli.CommandLineParser
;
import
org.apache.commons.cli.GnuParser
;
import
org.apache.commons.cli.HelpFormatter
;
import
org.apache.commons.cli.OptionBuilder
;
import
org.apache.commons.cli.Options
;
import
org.apache.commons.cli.ParseException
;
import
org.apache.hadoop.hbase.client.HTableInterface
;
import
org.apache.hadoop.hbase.client.HTablePool
;
import
org.apache.hadoop.hbase.client.Result
;
import
org.apache.hadoop.hbase.client.ResultScanner
;
import
org.apache.hadoop.hbase.client.Scan
;
import
org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter
;
import
org.apache.hadoop.util.ToolRunner
;
import
org.apache.log4j.Logger
;
import
org.warcbase.browser.WarcBrowser
;
import
org.warcbase.data.UrlUtil
;
import
tl.lin.data.SortableEntries.Order
;
import
tl.lin.data.fd.Object2IntFrequencyDistribution
;
import
tl.lin.data.fd.Object2IntFrequencyDistributionEntry
;
import
tl.lin.data.pair.PairOfObjectInt
;
public
class
PrintAllUris
{
private
static
final
Logger
LOG
=
Logger
.
getLogger
(
PrintAllUris
.
class
);
private
static
final
String
NAME_OPTION
=
"name"
;
private
static
final
String
PORT_OPTION
=
"port"
;
private
static
final
String
SERVER_OPTION
=
"server"
;
@SuppressWarnings
(
"static-access"
)
public
static
void
main
(
String
[]
args
)
throws
IOException
{
Options
options
=
new
Options
();
options
.
addOption
(
OptionBuilder
.
withArgName
(
"name"
).
hasArg
()
.
withDescription
(
"name of the archive"
).
create
(
NAME_OPTION
));
options
.
addOption
(
OptionBuilder
.
withArgName
(
"num"
).
hasArg
().
withDescription
(
"port to serve on"
)
.
create
(
PORT_OPTION
));
options
.
addOption
(
OptionBuilder
.
withArgName
(
"url"
).
hasArg
().
withDescription
(
"server prefix"
)
.
create
(
SERVER_OPTION
));
CommandLine
cmdline
=
null
;
CommandLineParser
parser
=
new
GnuParser
();
try
{
cmdline
=
parser
.
parse
(
options
,
args
);
}
catch
(
ParseException
exp
)
{
System
.
err
.
println
(
"Error parsing command line: "
+
exp
.
getMessage
());
System
.
exit
(-
1
);
}
if
(!
cmdline
.
hasOption
(
NAME_OPTION
))
{
HelpFormatter
formatter
=
new
HelpFormatter
();
formatter
.
printHelp
(
CountRowTypes
.
class
.
getCanonicalName
(),
options
);
System
.
exit
(-
1
);
}
String
name
=
cmdline
.
getOptionValue
(
NAME_OPTION
);
if
(!
cmdline
.
hasOption
(
PORT_OPTION
)
||
!
cmdline
.
hasOption
(
SERVER_OPTION
))
{
HelpFormatter
formatter
=
new
HelpFormatter
();
formatter
.
printHelp
(
WarcBrowser
.
class
.
getClass
().
getName
(),
options
);
ToolRunner
.
printGenericCommandUsage
(
System
.
out
);
System
.
exit
(-
1
);
}
int
port
=
Integer
.
parseInt
(
cmdline
.
getOptionValue
(
PORT_OPTION
));
String
server
=
cmdline
.
getOptionValue
(
SERVER_OPTION
);
HTablePool
pool
=
new
HTablePool
();
HTableInterface
table
=
pool
.
getTable
(
name
);
Scan
scan
=
new
Scan
();
scan
.
setFilter
(
new
FirstKeyOnlyFilter
());
ResultScanner
scanner
=
null
;
scanner
=
table
.
getScanner
(
scan
);
Object2IntFrequencyDistribution
<
String
>
domainCounter
=
new
Object2IntFrequencyDistributionEntry
<
String
>();
String
url
=
null
;
for
(
Result
rr
=
scanner
.
next
();
rr
!=
null
;
rr
=
scanner
.
next
())
{
byte
[]
key
=
rr
.
getRow
();
url
=
new
String
(
key
,
"UTF8"
);
url
=
UrlUtil
.
getDomain
(
url
);
url
=
UrlUtil
.
reverseHostname
(
url
);
domainCounter
.
increment
(
url
);
}
PrintWriter
writer
=
new
PrintWriter
(
"urls.html"
,
"UTF-8"
);
writer
.
println
(
"<html>"
);
writer
.
println
(
"<body>"
);
for
(
PairOfObjectInt
<
String
>
entry
:
domainCounter
.
getEntries
(
Order
.
ByRightElementDescending
))
{
writer
.
println
(
"<a href=\""
+
server
+
name
+
"?query="
+
entry
.
getLeftElement
()
+
"/"
+
"\">"
+
entry
.
getLeftElement
()
+
"</a><br/>"
);
}
writer
.
println
(
"</body>"
);
writer
.
println
(
"</html>"
);
writer
.
close
();
pool
.
close
();
}
}
Event Timeline
Log In to Comment