org.wandora.application.tools.extractors.reddit

Class AbstractRedditExtractor

java.lang.Object

org.wandora.application.tools.AbstractWandoraTool

org.wandora.application.tools.extractors.AbstractExtractor

org.wandora.application.tools.extractors.reddit.AbstractRedditExtractor

All Implemented Interfaces:

java.io.Serializable, java.lang.Runnable, BrowserPluginExtractor, DropExtractor, WandoraTool, WandoraToolLogger, Handler, InterruptHandler, TopicMapLogger

Direct Known Subclasses:

RedditThingExtractor

public abstract class AbstractRedditExtractor
extends AbstractExtractor

Author:

Eero Lehtonen, akivela

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
private static java.lang.String	ACC_ACT_SI
private static java.lang.String	ACCOUNT_SI
private static java.util.Map<java.lang.Integer,java.lang.String>	additionalPhrases
protected static java.lang.String	apiRoot
private static java.lang.String	AWARD_SI
private static java.lang.String	BODY_SI
private static java.lang.String	CHILD_SI
private static java.lang.String	CKARMA_SI
private static java.lang.String	COMMENT_SI
private java.lang.String[]	contentTypes
private static java.util.HashMap<java.lang.String,java.lang.Boolean>	CRAWL_SETTINGS
private static java.lang.String	CREATED_SI
private static java.lang.String	CREATED_U_SI
private static java.text.DateFormat	dateTimeFormat
private static java.lang.String	DESC_SI
private static java.lang.String	DESTINATION_SI
private static java.lang.String	DOWN_SI
private static java.lang.String	ERROR_MSG
private static java.lang.String	ERROR_RESPONSE_MSG
private static java.util.ArrayList<java.lang.String>	extracted
private static java.lang.String	LANG_SI
private static java.lang.String	LINK_SI
private static java.lang.String	LKARMA_SI
private static java.lang.String	MESSAGE_SI
private static java.lang.String	PAR_CHILD_SI
private static java.lang.String	PARENT_SI
private static int	progress
private static java.lang.String	PROMO_SI
private static java.lang.String	PUB_DESC_SI
protected Requester	requester
private static java.lang.String	SCORE_SI
private static java.lang.String	SI_ROOT
private static java.lang.String	SUBREDDIT_SI
private static java.lang.String	SUBS_SI
private static java.lang.String	THING_TYPE_ACCOUNT
private static java.lang.String	THING_TYPE_AWARD
private static java.lang.String	THING_TYPE_COMMENT
private static java.lang.String	THING_TYPE_LINK
private static java.lang.String	THING_TYPE_MESSAGE
private static java.lang.String	THING_TYPE_MORE
private static java.lang.String	THING_TYPE_PROMO
private static java.lang.String	THING_TYPE_SUBREDDIT
private static java.lang.String	TITLE_SI
private static java.lang.String	UP_SI

Fields inherited from class org.wandora.application.tools.extractors.AbstractExtractor

CUSTOM_EXTRACTOR, DONE_FAILED, DONE_MANY, DONE_ONE, EXACTLY_GIVEN_URLS, FILE_EXTRACTOR, FILE_PATTERN, GIVEN_URLS_AND_ALL_CRAWLED_DOCUMENTS, GIVEN_URLS_AND_CRAWLED_DOCUMENTS_IN_URL_DOMAIN, GIVEN_URLS_AND_LINKED_DOCUMENTS, GIVEN_URLS_AND_URL_BELOW, INFO_WAIT_WHILE_WORKING, LOG_TITLE, POINT_START_URL_TEXT, RAW_EXTRACTOR, SELECT_DIALOG_TITLE, STRING_EXTRACTOR_NOT_SUPPORTED_MESSAGE, URL_EXTRACTOR

Fields inherited from interface org.wandora.application.WandoraToolLogger

CLOSE, EXECUTE, INVISIBLE, VISIBLE, WAIT

Fields inherited from interface org.wandora.application.tools.browserextractors.BrowserPluginExtractor

RETURN_ERROR, RETURN_INFO

Constructor Summary

Constructors

Constructor and Description
AbstractRedditExtractor()

Method Summary

Modifier and Type	Method and Description
private void	addAccountOccurenceData(TopicMap tm, org.wandora.dep.json.JSONObject accountData, Topic accountTopic)
private void	addCommentOccurenceData(TopicMap tm, org.wandora.dep.json.JSONObject commentData, Topic commentTopic)
private void	addLinkOccurenceData(TopicMap tm, org.wandora.dep.json.JSONObject linkData, Topic linkTopic)
private void	addSubredditOccurrenceData(TopicMap tm, org.wandora.dep.json.JSONObject subredditData, Topic subredditTopic)
private void	associateAccount(TopicMap tm, org.wandora.dep.json.JSONObject thing, Topic account, java.util.HashMap<java.lang.String,Topic> types)
private void	associateParent(TopicMap tm, Topic commentTopic, Topic parentTopic)
private void	associateSubreddit(TopicMap tm, org.wandora.dep.json.JSONObject thing, Topic subreddit, java.util.HashMap<java.lang.String,Topic> types)
protected static java.util.HashMap<java.lang.String,Topic>	getAssociationTypes(TopicMap tm)
java.lang.String[]	getContentTypes() Returns an array of String containing the content-types this ContentHandler can process.
java.lang.String	getDescription() AdminToolManager views tool descriptions while user browses available tools and build user customizable GUI elements such as Tools menu.
javax.swing.Icon	getIcon() All tools may have identifying graphic icon used within tool GUI elements.
protected static Topic	getLangTopic(TopicMap tm)
java.lang.String	getName() Tools name represent the tool in UI unless the tool has been given explicitly another GUI name.
protected static Topic	getOrCreateTopic(TopicMap tm, java.lang.String si)
protected static Topic	getOrCreateTopic(TopicMap tm, java.lang.String si, java.lang.String bn)
protected static Topic	getRedditClass(TopicMap tm)
protected static void	getSubmissions(java.lang.String q, ParseCallback<com.mashape.unirest.http.JsonNode> callback)
protected static void	getSubreddits(java.lang.String q, ParseCallback<com.mashape.unirest.http.JsonNode> callback)
protected static java.util.HashMap<java.lang.String,Topic>	getThingTypes(TopicMap tm)
protected static Topic	getWandoraClassTopic(TopicMap tm)
protected void	logParseCallbackError(java.lang.Exception e, java.lang.String body)
protected void	logParseCallbackRunError(java.lang.Exception e)
protected static void	makeSubclassOf(TopicMap tm, Topic t, Topic superclass)
private Topic	parseAccount(org.wandora.dep.json.JSONObject account, java.util.HashMap<java.lang.String,Topic> thingTypes, TopicMap tm)
private Topic	parseComment(org.wandora.dep.json.JSONObject c, java.util.HashMap<java.lang.String,Topic> thingTypes, TopicMap tm)
private Topic	parseLink(org.wandora.dep.json.JSONObject l, java.util.HashMap<java.lang.String,Topic> thingTypes, TopicMap tm)
private void	parseMore(org.wandora.dep.json.JSONObject commentData, org.wandora.dep.json.JSONObject child, TopicMap tm, java.util.HashMap<java.lang.String,Topic> thingTypes)
private Topic	parseSubreddit(org.wandora.dep.json.JSONObject subreddit, java.util.HashMap<java.lang.String,Topic> thingTypes, TopicMap tm)
protected void	parseThing(org.wandora.dep.json.JSONObject thing, TopicMap tm, java.util.HashMap<java.lang.String,Topic> thingTypes)
protected void	parseThing(org.wandora.dep.json.JSONObject thing, TopicMap tm, java.util.HashMap<java.lang.String,Topic> thingTypes, java.util.HashMap<java.lang.String,java.lang.Boolean> crawlSettings)
static void	resetExtracted()
boolean	runInOwnThread() Whether or not this tool should fork own thread.
protected static java.lang.String	statusToPhrase(int status)
protected static java.lang.String	unixToString(long unixTimeStamp)
boolean	useURLCrawler()

Methods inherited from class org.wandora.application.tools.extractors.AbstractExtractor

_extractTopicsFrom, _extractTopicsFrom, _extractTopicsFrom, acceptBrowserExtractRequest, addCrawlerUrl, browserExtractorConsumesPlainText, buildSI, buildSL, clearMasterSubject, createAssociation, createAssociation, createTopic, createTopic, createTopic, createTopic, createTopic, createTopic, createTopic, croppedFilename, croppedFilename, croppedUrlString, croppedUrlString, doBrowserExtract, dropExtract, dropExtract, dropExtract, execute, extractTopicsFrom, extractTopicsFrom, extractTopicsFrom, extractTopicsFrom, extractTopicsFromText, getBrowserExtractorName, getCrawlerMode, getExtractorType, getForceContent, getForceFiles, getForceUrls, getGUIText, getGUIText, getInterruptsHandled, getMasterSubject, getType, getWandora, handle, handleContent, handleCustomType, handleFiles, handleForcedContent, handleInterrupt, handleStringContent, handleUrls, initializeCustomType, instantDropHandle, makeSubclassOfWandoraClass, setData, setDisplayName, setForceContent, setForceFiles, setForceUrls, setMasterSubject, setMasterSubject, setTopicMap, setupCrawler, setWandora, takeNap, urlEncode, useTempTopicMap

Methods inherited from class org.wandora.application.tools.AbstractWandoraTool

addUndoMarker, addUndoMarker, allowMultipleInvocations, clearAllThreads, clearThreads, clearThreads, clearToolLock, clearToolLock, clearToolLocks, configure, execute, execute, forceStop, forceStop, getContext, getCurrentLogger, getDefaultLogger, getHistory, getLastLogger, getState, getThreads, getThreads, getToolMenuItem, getToolMenuItem, getTopicName, hlog, initialize, interruptAllThreads, interruptThreads, interruptThreads, isConfigurable, isRunning, isRunning, lockLog, log, log, log, log, requiresRefresh, run, setContext, setDefaultLogger, setLogTitle, setProgress, setProgressMax, setState, setToolLogger, singleLog, singleLog, singleLog, solveContextTopicMap, solveNameForTopicMap, writeOptions

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.wandora.application.WandoraTool

configure, execute, execute, getContext, getToolMenuItem, hlog, initialize, isConfigurable, isRunning, log, log, log, log, requiresRefresh, setContext, setToolLogger, writeOptions

Methods inherited from interface org.wandora.application.WandoraToolLogger

forceStop, getHistory, getState, lockLog, setLogTitle, setProgress, setProgressMax, setState

Methods inherited from interface java.lang.Runnable

run

Field Detail

ERROR_MSG

private static final java.lang.String ERROR_MSG

See Also:

Constant Field Values

ERROR_RESPONSE_MSG

private static final java.lang.String ERROR_RESPONSE_MSG

See Also:

Constant Field Values

apiRoot

protected static final java.lang.String apiRoot

See Also:

Constant Field Values

THING_TYPE_COMMENT

private static final java.lang.String THING_TYPE_COMMENT

See Also:

Constant Field Values

THING_TYPE_ACCOUNT

private static final java.lang.String THING_TYPE_ACCOUNT

See Also:

Constant Field Values

THING_TYPE_LINK

private static final java.lang.String THING_TYPE_LINK

See Also:

Constant Field Values

THING_TYPE_MESSAGE

private static final java.lang.String THING_TYPE_MESSAGE

See Also:

Constant Field Values

THING_TYPE_SUBREDDIT

private static final java.lang.String THING_TYPE_SUBREDDIT

See Also:

Constant Field Values

THING_TYPE_AWARD

private static final java.lang.String THING_TYPE_AWARD

See Also:

Constant Field Values

THING_TYPE_PROMO

private static final java.lang.String THING_TYPE_PROMO

See Also:

Constant Field Values

THING_TYPE_MORE

private static final java.lang.String THING_TYPE_MORE

See Also:

Constant Field Values

LANG_SI

private static final java.lang.String LANG_SI

See Also:

Constant Field Values

SI_ROOT

private static final java.lang.String SI_ROOT

See Also:

Constant Field Values

COMMENT_SI

private static final java.lang.String COMMENT_SI

See Also:

Constant Field Values

LINK_SI

private static final java.lang.String LINK_SI

See Also:

Constant Field Values

ACCOUNT_SI

private static final java.lang.String ACCOUNT_SI

See Also:

Constant Field Values

MESSAGE_SI

private static final java.lang.String MESSAGE_SI

See Also:

Constant Field Values

SUBREDDIT_SI

private static final java.lang.String SUBREDDIT_SI

See Also:

Constant Field Values

AWARD_SI

private static final java.lang.String AWARD_SI

See Also:

Constant Field Values

PROMO_SI

private static final java.lang.String PROMO_SI

See Also:

Constant Field Values

PARENT_SI

private static final java.lang.String PARENT_SI

See Also:

Constant Field Values

CHILD_SI

private static final java.lang.String CHILD_SI

See Also:

Constant Field Values

PAR_CHILD_SI

private static final java.lang.String PAR_CHILD_SI

See Also:

Constant Field Values

DESTINATION_SI

private static final java.lang.String DESTINATION_SI

See Also:

Constant Field Values

CREATED_SI

private static final java.lang.String CREATED_SI

See Also:

Constant Field Values

CREATED_U_SI

private static final java.lang.String CREATED_U_SI

See Also:

Constant Field Values

BODY_SI

private static final java.lang.String BODY_SI

See Also:

Constant Field Values

UP_SI

private static final java.lang.String UP_SI

See Also:

Constant Field Values

DOWN_SI

private static final java.lang.String DOWN_SI

See Also:

Constant Field Values

SCORE_SI

private static final java.lang.String SCORE_SI

See Also:

Constant Field Values

CKARMA_SI

private static final java.lang.String CKARMA_SI

See Also:

Constant Field Values

LKARMA_SI

private static final java.lang.String LKARMA_SI

See Also:

Constant Field Values

DESC_SI

private static final java.lang.String DESC_SI

See Also:

Constant Field Values

PUB_DESC_SI

private static final java.lang.String PUB_DESC_SI

See Also:

Constant Field Values

ACC_ACT_SI

private static final java.lang.String ACC_ACT_SI

See Also:

Constant Field Values

SUBS_SI

private static final java.lang.String SUBS_SI

See Also:

Constant Field Values

TITLE_SI

private static final java.lang.String TITLE_SI

See Also:

Constant Field Values

CRAWL_SETTINGS

private static java.util.HashMap<java.lang.String,java.lang.Boolean> CRAWL_SETTINGS

dateTimeFormat

private static java.text.DateFormat dateTimeFormat

extracted

private static java.util.ArrayList<java.lang.String> extracted

progress

private static int progress

requester

protected Requester requester

contentTypes

private final java.lang.String[] contentTypes

additionalPhrases

private static final java.util.Map<java.lang.Integer,java.lang.String> additionalPhrases

Constructor Detail

AbstractRedditExtractor

public AbstractRedditExtractor()

Method Detail

getName

public java.lang.String getName()

Description copied from class: AbstractWandoraTool

Tools name represent the tool in UI unless the tool has been given explicitly another GUI name. All tools should override this method and return a valid string representing name of the tool.

Specified by:

getName in interface WandoraTool

Overrides:

getName in class AbstractExtractor

Returns:

String representing tool's name.

getDescription

public java.lang.String getDescription()

Description copied from class: AbstractWandoraTool

AdminToolManager views tool descriptions while user browses available tools and build user customizable GUI elements such as Tools menu. By default description equals the tool name. All tools should override this method.

Specified by:

getDescription in interface WandoraTool

Overrides:

getDescription in class AbstractExtractor

Returns:

String representing tool's description.

getIcon

public javax.swing.Icon getIcon()

Description copied from class: AbstractWandoraTool

All tools may have identifying graphic icon used within tool GUI elements. getIcon should return Icon object of the tool.

Specified by:

getIcon in interface WandoraTool

Overrides:

getIcon in class AbstractExtractor

Returns:

icon for the tool.

getContentTypes

public java.lang.String[] getContentTypes()

Description copied from interface: Handler

Returns an array of String containing the content-types this ContentHandler can process.

Specified by:

getContentTypes in interface Handler

Overrides:

getContentTypes in class AbstractExtractor

Returns:

String array representing content types.

useURLCrawler

public boolean useURLCrawler()

Overrides:

useURLCrawler in class AbstractExtractor

runInOwnThread

public boolean runInOwnThread()

Description copied from class: AbstractWandoraTool

Whether or not this tool should fork own thread. If own thread is allowed, the execution of the tool return immediately. If own thread is not allowed the thread entering initial execute method is used. Extending classes should override this method.

Overrides:

runInOwnThread in class AbstractExtractor

Returns:

true if the tool should should be ran in separate thread.

resetExtracted

public static void resetExtracted()

statusToPhrase

protected static java.lang.String statusToPhrase(int status)

getRedditClass

protected static Topic getRedditClass(TopicMap tm)
                               throws TopicMapException

Throws:

TopicMapException

getWandoraClassTopic

protected static Topic getWandoraClassTopic(TopicMap tm)
                                     throws TopicMapException

Throws:

TopicMapException

getOrCreateTopic

protected static Topic getOrCreateTopic(TopicMap tm,
                                        java.lang.String si)
                                 throws TopicMapException

Throws:

TopicMapException

getOrCreateTopic

protected static Topic getOrCreateTopic(TopicMap tm,
                                        java.lang.String si,
                                        java.lang.String bn)
                                 throws TopicMapException

Throws:

TopicMapException

makeSubclassOf

protected static void makeSubclassOf(TopicMap tm,
                                     Topic t,
                                     Topic superclass)
                              throws TopicMapException

Throws:

TopicMapException

getLangTopic

protected static Topic getLangTopic(TopicMap tm)
                             throws TopicMapException

Throws:

TopicMapException

unixToString

protected static java.lang.String unixToString(long unixTimeStamp)

getThingTypes

protected static java.util.HashMap<java.lang.String,Topic> getThingTypes(TopicMap tm)
                                                                  throws TopicMapException

Throws:

TopicMapException

getAssociationTypes

protected static java.util.HashMap<java.lang.String,Topic> getAssociationTypes(TopicMap tm)
                                                                        throws TopicMapException

Throws:

TopicMapException

associateParent

private void associateParent(TopicMap tm,
                             Topic commentTopic,
                             Topic parentTopic)
                      throws TopicMapException

Throws:

TopicMapException

associateAccount

private void associateAccount(TopicMap tm,
                              org.wandora.dep.json.JSONObject thing,
                              Topic account,
                              java.util.HashMap<java.lang.String,Topic> types)
                       throws TopicMapException,
                              org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

associateSubreddit

private void associateSubreddit(TopicMap tm,
                                org.wandora.dep.json.JSONObject thing,
                                Topic subreddit,
                                java.util.HashMap<java.lang.String,Topic> types)
                         throws TopicMapException,
                                org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

addLinkOccurenceData

private void addLinkOccurenceData(TopicMap tm,
                                  org.wandora.dep.json.JSONObject linkData,
                                  Topic linkTopic)
                           throws TopicMapException,
                                  org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

addCommentOccurenceData

private void addCommentOccurenceData(TopicMap tm,
                                     org.wandora.dep.json.JSONObject commentData,
                                     Topic commentTopic)
                              throws TopicMapException,
                                     org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

addAccountOccurenceData

private void addAccountOccurenceData(TopicMap tm,
                                     org.wandora.dep.json.JSONObject accountData,
                                     Topic accountTopic)
                              throws TopicMapException,
                                     org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

addSubredditOccurrenceData

private void addSubredditOccurrenceData(TopicMap tm,
                                        org.wandora.dep.json.JSONObject subredditData,
                                        Topic subredditTopic)
                                 throws TopicMapException,
                                        org.wandora.dep.json.JSONException

Throws:

TopicMapException

org.wandora.dep.json.JSONException

getSubmissions

protected static void getSubmissions(java.lang.String q,
                                     ParseCallback<com.mashape.unirest.http.JsonNode> callback)

getSubreddits

protected static void getSubreddits(java.lang.String q,
                                    ParseCallback<com.mashape.unirest.http.JsonNode> callback)

parseThing

protected void parseThing(org.wandora.dep.json.JSONObject thing,
                          TopicMap tm,
                          java.util.HashMap<java.lang.String,Topic> thingTypes,
                          java.util.HashMap<java.lang.String,java.lang.Boolean> crawlSettings)

parseThing

protected void parseThing(org.wandora.dep.json.JSONObject thing,
                          TopicMap tm,
                          java.util.HashMap<java.lang.String,Topic> thingTypes)

parseMore

private void parseMore(org.wandora.dep.json.JSONObject commentData,
                       org.wandora.dep.json.JSONObject child,
                       TopicMap tm,
                       java.util.HashMap<java.lang.String,Topic> thingTypes)
                throws org.wandora.dep.json.JSONException,
                       TopicMapException

Throws:

org.wandora.dep.json.JSONException

TopicMapException

parseLink

private Topic parseLink(org.wandora.dep.json.JSONObject l,
                        java.util.HashMap<java.lang.String,Topic> thingTypes,
                        TopicMap tm)
                 throws org.wandora.dep.json.JSONException,
                        TopicMapException

Throws:

org.wandora.dep.json.JSONException

TopicMapException

parseComment

private Topic parseComment(org.wandora.dep.json.JSONObject c,
                           java.util.HashMap<java.lang.String,Topic> thingTypes,
                           TopicMap tm)
                    throws org.wandora.dep.json.JSONException,
                           TopicMapException

Throws:

org.wandora.dep.json.JSONException

TopicMapException

parseAccount

private Topic parseAccount(org.wandora.dep.json.JSONObject account,
                           java.util.HashMap<java.lang.String,Topic> thingTypes,
                           TopicMap tm)
                    throws org.wandora.dep.json.JSONException,
                           TopicMapException

Throws:

org.wandora.dep.json.JSONException

TopicMapException

parseSubreddit

private Topic parseSubreddit(org.wandora.dep.json.JSONObject subreddit,
                             java.util.HashMap<java.lang.String,Topic> thingTypes,
                             TopicMap tm)
                      throws org.wandora.dep.json.JSONException,
                             TopicMapException

Throws:

org.wandora.dep.json.JSONException

TopicMapException

logParseCallbackError

protected void logParseCallbackError(java.lang.Exception e,
                                     java.lang.String body)

logParseCallbackRunError

protected void logParseCallbackRunError(java.lang.Exception e)