\documentclass[a4paper]{book}
\usepackage[times,inconsolata,hyper]{Rd}
\usepackage{makeidx}
\usepackage[utf8,latin1]{inputenc}
% \usepackage{graphicx} % @USE GRAPHICX@
\makeindex{}
\begin{document}
\chapter*{}
\begin{center}
{\textbf{\huge Package `tm.plugin.webmining'}}
\par\bigskip{\large \today}
\end{center}
\begin{description}
\raggedright{}
\item[Version]\AsIs{1.3}
\item[Date]\AsIs{2015-05-07}
\item[Title]\AsIs{Retrieve Structured, Textual Data from Various Web Sources}
\item[Depends]\AsIs{R (>= 3.1.0)}
\item[Imports]\AsIs{NLP (>= 0.1-2), tm (>= 0.6), boilerpipeR, RCurl, XML, RJSONIO}
\item[Suggests]\AsIs{testthat}
\item[Description]\AsIs{Facilitate text retrieval from feed
formats like XML (RSS, ATOM) and JSON. Also direct retrieval from
HTML is supported. As most (news) feeds only incorporate small
fractions of the original text tm.plugin.webmining even retrieves
and extracts the text of the original text source.}
\item[License]\AsIs{GPL-3}
\item[URL]\AsIs{}\url{https://github.com/mannau/tm.plugin.webmining}\AsIs{}
\item[BugReports]\AsIs{}\url{https://github.com/mannau/tm.plugin.webmining/issues}\AsIs{}
\item[NeedsCompilation]\AsIs{no}
\item[Author]\AsIs{Mario Annau [aut, cre]}
\item[Maintainer]\AsIs{Mario Annau }\email{mario.annau@gmail.com}\AsIs{}
\item[Repository]\AsIs{CRAN}
\item[Date/Publication]\AsIs{2015-05-11 00:20:43}
\end{description}
\Rdcontents{\R{} topics documented:}
\inputencoding{utf8}
\HeaderA{tm.plugin.webmining-package}{Retrieve structured, textual data from various web sources}{tm.plugin.webmining.Rdash.package}
\aliasA{tm.plugin.webmining}{tm.plugin.webmining-package}{tm.plugin.webmining}
\aliasA{webmining}{tm.plugin.webmining-package}{webmining}
\keyword{package}{tm.plugin.webmining-package}
%
\begin{Description}\relax
tm.plugin.webmining facilitates the retrieval of textual data through various
web feed formats like XML and JSON. Also direct retrieval from HTML
is supported. As most (news) feeds only incorporate small fractions
of the original text tm.plugin.webmining goes a step further and even
retrieves and extracts the text of the original text source.
Generally, the retrieval procedure can be described as a two--step process:
\begin{description}

\item[Meta Retrieval] In a first step, all relevant meta feeds are retrieved.
From these feeds all relevant meta data items are extracted.

\item[Content Retrieval] In a second step the relevant source content is retrieved.
Using the \code{boilerpipeR} package even the main content of \code{HTML} pages can
be extracted.

\end{description}

\end{Description}
%
\begin{Author}\relax
Mario Annau \email{mario.annau@gmail}
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebCorpus}{WebCorpus}} \code{\LinkA{GoogleFinanceSource}{GoogleFinanceSource}} \code{\LinkA{GoogleNewsSource}{GoogleNewsSource}} \code{\LinkA{NYTimesSource}{NYTimesSource}} \code{\LinkA{ReutersNewsSource}{ReutersNewsSource}} \code{\LinkA{YahooFinanceSource}{YahooFinanceSource}} \code{\LinkA{YahooInplaySource}{YahooInplaySource}} \code{\LinkA{YahooNewsSource}{YahooNewsSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
googlefinance <- WebCorpus(GoogleFinanceSource("NASDAQ:MSFT"))
googlenews <- WebCorpus(GoogleNewsSource("Microsoft"))
nytimes <- WebCorpus(NYTimesSource("Microsoft", appid = nytimes_appid))
reutersnews <- WebCorpus(ReutersNewsSource("businessNews"))
yahoofinance <- WebCorpus(YahooFinanceSource("MSFT"))
yahooinplay <- WebCorpus(YahooInplaySource())
yahoonews <- WebCorpus(YahooNewsSource("Microsoft"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{corpus.update}{Update/Extend \code{\LinkA{WebCorpus}{WebCorpus}} with new feed items.}{corpus.update}
\methaliasA{corpus.update.WebCorpus}{corpus.update}{corpus.update.WebCorpus}
%
\begin{Description}\relax
The \code{corpus.update} method ensures, that the original
\code{\LinkA{WebCorpus}{WebCorpus}} feed sources are downloaded and checked against
already included \code{TextDocument}s. Based on the \code{ID} included
in the  \code{TextDocument}'s meta data, only new feed elements are
downloaded and added to the \code{\LinkA{WebCorpus}{WebCorpus}}.
All relevant information regariding the original source feeds are stored
in the \code{\LinkA{WebCorpus}{WebCorpus}}' meta data (\code{\LinkA{meta}{meta}}).
\end{Description}
%
\begin{Usage}
\begin{verbatim}
corpus.update(x, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] object of type \code{\LinkA{WebCorpus}{WebCorpus}}

\item[\code{...}] \begin{description}

\item[fieldname] name of \code{\LinkA{Corpus}{Corpus}} field name to be used as ID, defaults to "ID"
\item[retryempty] specifies if empty corpus elements should be downloaded again, defaults to TRUE
\item[...] additional parameters to \code{\LinkA{Corpus}{Corpus}} function

\end{description}

\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{encloseHTML}{Enclose Text Content in HTML tags}{encloseHTML}
\methaliasA{encloseHTML.character}{encloseHTML}{encloseHTML.character}
\methaliasA{encloseHTML.PlainTextDocument}{encloseHTML}{encloseHTML.PlainTextDocument}
%
\begin{Description}\relax
Simple helper function which encloses text content of character
(or \code{\LinkA{TextDocument}{TextDocument}}) in HTML-tags. That way, HTML
content can be easier parsed by \code{\LinkA{htmlTreeParse}{htmlTreeParse}}
\end{Description}
%
\begin{Usage}
\begin{verbatim}
encloseHTML(x)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] object of PlainTextDocument class
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{extract}{Extract main content from \code{TextDocument}s.}{extract}
\methaliasA{extract.PlainTextDocument}{extract}{extract.PlainTextDocument}
%
\begin{Description}\relax
Use implemented extraction functions (through boilerpipeR) to extract main content from
\code{TextDocument}s.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
extract(x, extractor, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] PlainTextDocument

\item[\code{extractor}] default extraction function to be used, defaults to \code{\LinkA{extractContentDOM}{extractContentDOM}}

\item[\code{...}] additional parameters to extractor function
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{extractContentDOM}{Extract Main HTML Content from DOM}{extractContentDOM}
\aliasA{assignValues}{extractContentDOM}{assignValues}
\aliasA{calcDensity}{extractContentDOM}{calcDensity}
\aliasA{getMainText}{extractContentDOM}{getMainText}
\aliasA{removeTags}{extractContentDOM}{removeTags}
%
\begin{Description}\relax
Function extracts main HTML Content using its Document Object Model.
Idea comes basically from the fact, that main content of an HTML Document
is in a subnode of the HTML DOM Tree with a high text-to-tag ratio.
Internally, this function also calls
\code{assignValues}, \code{calcDensity}, \code{getMainText}
and \code{removeTags}.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
extractContentDOM(url, threshold, asText = TRUE, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{url}] character, url or filename

\item[\code{threshold}] threshold for extraction, defaults to 0.5

\item[\code{asText}] boolean, specifies if url should be interpreted as character

\item[\code{...}] Additional Parameters to \code{\LinkA{htmlTreeParse}{htmlTreeParse}}
\end{ldescription}
\end{Arguments}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{References}\relax
\url{http://www.elias.cn/En/ExtMainText},
\url{http://ai-depot.com/articles/the-easy-way-to-extract-useful-text-from-arbitrary-html/}
\Cite{Gupta et al., DOM-based Content Extraction of HTML Documents},\url{http://www2003.org/cdrom/papers/refereed/p583/p583-gupta.html}
\end{References}
%
\begin{SeeAlso}\relax
\code{\LinkA{xmlNode}{xmlNode}}
\end{SeeAlso}
\inputencoding{utf8}
\HeaderA{extractHTMLStrip}{Simply strip HTML Tags from Document}{extractHTMLStrip}
%
\begin{Description}\relax
\code{extractHTMLStrip} parses an url, character or filename, reads the DOM
tree, removes all HTML tags in the tree and outputs the source text without
markup.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
extractHTMLStrip(url, asText = TRUE, encoding, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{url}] character, url or filename

\item[\code{asText}] specifies if url parameter is a \code{character}, defaults to TRUE

\item[\code{encoding}] specifies local encoding to be used, depending on platform

\item[\code{...}] Additional parameters for \code{\LinkA{htmlTreeParse}{htmlTreeParse}}
\end{ldescription}
\end{Arguments}
%
\begin{Note}\relax
Input text should be enclosed in <html>'TEXT'</html> tags to ensure correct
DOM parsing (issue especially under .Platform\$os.type = 'windows')
\end{Note}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{xmlNode}{xmlNode}}

\code{\LinkA{htmlTreeParse}{htmlTreeParse}} \code{\LinkA{encloseHTML}{encloseHTML}}
\end{SeeAlso}
\inputencoding{utf8}
\HeaderA{feedquery}{Buildup string for feedquery.}{feedquery}
%
\begin{Description}\relax
Function has partly been taken from \code{\LinkA{getForm}{getForm}} function.
Generally, a feed query is a string built up as follows: \\{}
\code{<url>?<param1=value1>\&<param2=value2>\&...\&<paramN=valueN>} \\{}
By specifying a feed url and parameter--value pairs (as list) we can easily
generate a feed query in R.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
feedquery(url, params)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{url}] character specifying feed url

\item[\code{params}] list which contains feed parameters, e.g. list(param1="value1", param2="value2")
\end{ldescription}
\end{Arguments}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{xmlNode}{xmlNode}} \code{\LinkA{getForm}{getForm}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
feedquery(url = "http://dummy.com",
params = list(param1 = "value1", param2 = "value2"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{getEmpty}{Retrieve Empty Corpus Elements through \code{\$postFUN}.}{getEmpty}
\methaliasA{getEmpty.WebCorpus}{getEmpty}{getEmpty.WebCorpus}
%
\begin{Description}\relax
Retrieve content of all empty (textlength equals zero) corpus elements. If
corpus element is empty, \code{\$postFUN} is called (specified in \code{\LinkA{meta}{meta}})
\end{Description}
%
\begin{Usage}
\begin{verbatim}
getEmpty(x, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] object of type \code{\LinkA{WebCorpus}{WebCorpus}}

\item[\code{...}] additional parameters to PostFUN
\end{ldescription}
\end{Arguments}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebCorpus}{WebCorpus}}
\end{SeeAlso}
\inputencoding{utf8}
\HeaderA{getLinkContent}{Get main content for corpus items, specified by links.}{getLinkContent}
%
\begin{Description}\relax
\code{getLinkContent} downloads and extracts content from weblinks for \code{\LinkA{Corpus}{Corpus}} objects.
Typically it is integrated and called as a post-processing function (field:\code{\$postFUN}) for most \code{\LinkA{WebSource}{WebSource}}
objects. \code{getLinkContent} implements content download in chunks which has been proven to be a stabler approach for
large content requests.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
getLinkContent(corpus, links = sapply(corpus, meta, "origin"),
  timeout.request = 30, chunksize = 20, verbose = getOption("verbose"),
  curlOpts = curlOptions(verbose = FALSE, followlocation = TRUE, maxconnects =
  5, maxredirs = 20, timeout = timeout.request, connecttimeout =
  timeout.request, ssl.verifyhost = FALSE, ssl.verifypeer = FALSE, useragent =
  "R", cookiejar = tempfile()), retry.empty = 3, sleep.time = 3,
  extractor = ArticleExtractor, .encoding = integer(), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{corpus}] object of class \code{\LinkA{Corpus}{Corpus}} for which link content should be downloaded

\item[\code{links}] character vector specifyinig links to be used for download, defaults to
sapply(corpus, meta, "Origin")

\item[\code{timeout.request}] timeout (in seconds) to be used for connections/requests, defaults to 30

\item[\code{chunksize}] Size of download chunks to be used for parallel retrieval, defaults to 20

\item[\code{verbose}] Specifies if retrieval info should be printed, defaults to getOption("verbose")

\item[\code{curlOpts}] curl options to be passed to \code{\LinkA{getURL}{getURL}}

\item[\code{retry.empty}] Specifies number of times empty content sites should be retried, defaults to 3

\item[\code{sleep.time}] Sleep time to be used between chunked download, defaults to 3 (seconds)

\item[\code{extractor}] Extractor to be used for content extraction, defaults to extractContentDOM

\item[\code{.encoding}] encoding to be used for \code{\LinkA{getURL}{getURL}}, defaults to integer() (=autodetect)

\item[\code{...}] additional parameters to \code{\LinkA{getURL}{getURL}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
corpus including downloaded link content
\end{Value}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}} \code{\LinkA{getURL}{getURL}} \code{\LinkA{Extractor}{Extractor}}
\end{SeeAlso}
\inputencoding{utf8}
\HeaderA{GoogleFinanceSource}{Get feed Meta Data from Google Finance.}{GoogleFinanceSource}
\aliasA{readGoogle}{GoogleFinanceSource}{readGoogle}
%
\begin{Description}\relax
Google Finance provides business and enterprise headlines for many companies. Coverage is
particularly strong for US-Markets. However, only up to 20 feed items can be retrieved.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
GoogleFinanceSource(query, params = list(hl = "en", q = query, ie = "utf-8",
  start = 0, num = 20, output = "rss"), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] ticker symbols of companies to be searched for, see \url{http://www.google.com/finance}.
Please note that Google ticker symbols need to be prefixed with the exchange name, e.g. NASDAQ:MSFT

\item[\code{params}] additional query parameters

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebXMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(GoogleFinanceSource("NASDAQ:MSFT"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{GoogleNewsSource}{Get feed data from Google News Search \url{http://news.google.com/}}{GoogleNewsSource}
%
\begin{Description}\relax
Google News Search is one of the most popular news aggregators on the web. News
can be retrieved for any customized user query. Up to 100 can be retrieved per
request.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
GoogleNewsSource(query, params = list(hl = "en", q = query, ie = "utf-8", num
  = 100, output = "rss"), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] Google News Search query

\item[\code{params,}] additional query parameters

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebXMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(GoogleNewsSource("Microsoft"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{NYTimesSource}{Get feed data from NYTimes Article Search (\url{http://developer.nytimes.com/docs/read/article\_search\_api\_v2}).}{NYTimesSource}
\aliasA{readNYTimes}{NYTimesSource}{readNYTimes}
%
\begin{Description}\relax
Excerpt from the website: "With the NYTimes Article Search API, you can search New York Times articles
from 1981 to today, retrieving headlines, abstracts, lead paragraphs, links to associated multimedia
and other article metadata. Along with standard keyword searching, the API also offers faceted searching.
The available facets include Times-specific fields such as sections, taxonomic classifiers and controlled
vocabulary terms (names of people, organizations and geographic locations)."
Feed retrieval is limited to 1000 items (or 100 pages).
\end{Description}
%
\begin{Usage}
\begin{verbatim}
NYTimesSource(query, n = 100, appid, count = 10, sleep = 1,
  params = list(format = "json", q = query, page = 1:ceiling(n/count),
  `api-key` = appid), curlOpts = curlOptions(followlocation = TRUE,
  maxconnects = 10, maxredirs = 10, timeout = 30, connecttimeout = 30), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] character specifying query to be used to search NYTimes articles

\item[\code{n}] number of items, defaults to 100

\item[\code{appid}] Developer App id to be used, obtained from \url{http://developer.nytimes.com/}

\item[\code{count}] number of results per page, defaults to 10

\item[\code{sleep}] integer; Seconds to sleep between feed retrieval.

\item[\code{params}] additional query parameters, specified as list, see \url{http://developer.nytimes.com/docs/read/article_search_api}

\item[\code{curlOpts}] CURLOptions; RCurl options used for feed retrieval.

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}, \code{\LinkA{readNYTimes}{readNYTimes}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
#nytimes_appid needs to be specified
corpus <- WebCorpus(NYTimesSource("Microsoft", appid = nytimes_appid))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{nytimes\_appid}{AppID for the NYtimes-API.}{nytimes.Rul.appid}
\keyword{data}{nytimes\_appid}
%
\begin{Description}\relax
USED ONLY FOR PACKAGE TESTING. PLEASE DOWNLOAD YOUR OWN KEY AT \url{http://developer.nytimes.com/}!!!
\end{Description}
%
\begin{Author}\relax
Mario Annau
\end{Author}
\inputencoding{utf8}
\HeaderA{parse}{Wrapper/Convenience function to ensure right encoding for different Platforms}{parse}
%
\begin{Description}\relax
Depending on specified type one of the following parser functions is called:
\begin{description}

\item[XML] \code{\LinkA{xmlInternalTreeParse}{xmlInternalTreeParse}}
\item[HTML] \code{\LinkA{htmlTreeParse}{htmlTreeParse}}
\item[JSON] \code{\LinkA{fromJSON}{fromJSON}}

\end{description}

\end{Description}
%
\begin{Usage}
\begin{verbatim}
parse(..., asText = TRUE, type = c("XML", "HTML", "JSON"))
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{...}] arguments to be passed to specified parser function

\item[\code{asText}] defines if input should be treated as text/character, default to TRUE

\item[\code{type}] either "XML", "HTML" or "JSON". Defaults to "XML"
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{readWeb}{Read content from WebXMLSource/WebHTMLSource/WebJSONSource.}{readWeb}
\aliasA{json\_content}{readWeb}{json.Rul.content}
\aliasA{readWebHTML}{readWeb}{readWebHTML}
\aliasA{readWebJSON}{readWeb}{readWebJSON}
\aliasA{readWebXML}{readWeb}{readWebXML}
%
\begin{Description}\relax
\code{readWeb} is a FunctionGenerator which specifies content retrieval from a \code{\LinkA{WebSource}{WebSource}}
content elements. Currently, it is defined for XML, HTML and JSON feeds through \code{readWebXML},
\code{readWebHTML} and \code{readWebJSON}. Also content parsers (\code{xml\_content}, \code{json\_content})
need to be defined.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
readWeb(spec, doc, parser, contentparser, freeFUN = NULL)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{spec}] specification of content reader

\item[\code{doc}] document to be parsed

\item[\code{parser}] parser function to be used

\item[\code{contentparser}] content parser function to be used, see also \code{tm:::xml\_content} or \code{json\_content}

\item[\code{freeFUN}] function to free memory from parsed object (actually only relevant for XML and HTML trees)
\end{ldescription}
\end{Arguments}
%
\begin{Value}
FunctionGenerator
\end{Value}
\inputencoding{utf8}
\HeaderA{removeNonASCII}{Remove non-ASCII characters from Text.}{removeNonASCII}
\methaliasA{removeNonASCII.PlainTextDocument}{removeNonASCII}{removeNonASCII.PlainTextDocument}
%
\begin{Description}\relax
This is a helper function to generate package data
without non-ASCII character and omit the warning at R CMD check.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
removeNonASCII(x, fields = c("Content", "Heading", "Description"),
  from = "UTF-8", to = "ASCII//TRANSLIT")
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] object of PlainTextDocument class

\item[\code{fields}] specifies fields to be converted, defaults to fields = c("Content", "Heading", "Description")

\item[\code{from}] specifies encoding from which conversion should be done, defaults to "UTF-8"

\item[\code{to}] speciefies target encoding, defaults to "ASCII//TRANSLIT"
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{ReutersNewsSource}{Get feed data from Reuters News RSS feed channels. Reuters provides numerous feed}{ReutersNewsSource}
\aliasA{readReutersNews}{ReutersNewsSource}{readReutersNews}
%
\begin{Description}\relax
channels (\url{http://www.reuters.com/tools/rss}) which can be retrieved through RSS
feeds. Only up to 25 items can be retrieved---therefore an alternative retrieval
through the Google Reader API (\code{link\{GoogleReaderSource\}}) could be considered.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
ReutersNewsSource(query = "businessNews", ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] Reuters News RSS Feed, see \url{http://www.reuters.com/tools/rss} for a list of all feeds provided. Note that only string after 'http://feeds.reuters.com/reuters/' must be given. Defaults to 'businessNews'.

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebXMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(ReutersNewsSource("businessNews"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{source.update}{Update WebXMLSource/WebHTMLSource/WebJSONSource}{source.update}
\methaliasA{source.update.WebHTMLSource}{source.update}{source.update.WebHTMLSource}
\methaliasA{source.update.WebJSONSource}{source.update}{source.update.WebJSONSource}
\methaliasA{source.update.WebXMLSource}{source.update}{source.update.WebXMLSource}
%
\begin{Description}\relax
Typically, update is called from \code{link\{corpus.update\}} and refreshes \code{\$Content} in
Source object.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
source.update(x)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] Source object to be updated
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{trimWhiteSpaces}{Trim White Spaces from Text Document.}{trimWhiteSpaces}
%
\begin{Description}\relax
Transformation function, actually equal to stripWhiteSpace
applicable for simple strings using Perl parser
\end{Description}
%
\begin{Usage}
\begin{verbatim}
trimWhiteSpaces(txt)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{txt}] character
\end{ldescription}
\end{Arguments}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{stripWhitespace}{stripWhitespace}}
\end{SeeAlso}
\inputencoding{utf8}
\HeaderA{WebCorpus}{WebCorpus constructor function.}{WebCorpus}
%
\begin{Description}\relax
\code{WebCorpus} adds further methods and meta data to \code{\LinkA{Corpus}{Corpus}} and therefore
constructs a derived class of \code{\LinkA{Corpus}{Corpus}}. Most importantly, \code{WebCorpus}
calls \code{\$PostFUN} on the generated \code{WebCorpus}, which retrieves the main content
for most implemented \code{WebSource}s. Thus it enables an efficient retrieval of new feed items
(\code{\LinkA{corpus.update}{corpus.update}}). All additional WebCorpus fields are added to \code{tm\$meta}
like \code{\$source}, \code{\$readerControl} and \code{\$postFUN}.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
WebCorpus(x, readerControl = list(reader = reader(x), language = "en"),
  postFUN = x$postFUN, retryEmpty = TRUE, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{x}] object of type Source, see also \code{\LinkA{Corpus}{Corpus}}

\item[\code{readerControl}] specifies reader to be used for \code{Source}, defaults to
list(reader = x\$DefaultReader, language = "en"

\item[\code{postFUN}] function to be applied to WebCorpus after web retrieval has been completed,
defaults to x\$PostFUN

\item[\code{retryEmpty}] specifies if retrieval for empty content elements should be repeated,
defaults to TRUE

\item[\code{...}] additional parameters for Corpus function (actually Corpus reader)
\end{ldescription}
\end{Arguments}
\inputencoding{utf8}
\HeaderA{WebSource}{Read Web Content and respective Link Content from feedurls.}{WebSource}
%
\begin{Description}\relax
WebSource is derived from \code{\LinkA{Source}{Source}}. In addition to calling the
base \code{\LinkA{Source}{Source}} constructor function it also retrieves the specified
feedurls and pre--parses the content with the parser function.
The fields \code{\$Content}, \code{\$Feedurls} \code{\$Parser} and \code{\$CurlOpts} are finally
added to the \code{Source} object.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
WebSource(feedurls, class = "WebXMLSource", reader, parser,
  encoding = "UTF-8", curlOpts = curlOptions(followlocation = TRUE,
  maxconnects = 20, maxredirs = 10, timeout = 30, connecttimeout = 30),
  postFUN = NULL, retrieveFeedURL = TRUE, ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{feedurls}] urls from feeds to be retrieved

\item[\code{class}] class label to be assigned to \code{Source} object, defaults to "WebXMLSource"

\item[\code{reader}] function to be used to read content, see also \code{\LinkA{readWeb}{readWeb}}

\item[\code{parser}] function to be used to split feed content into chunks, returns list of content elements

\item[\code{encoding}] specifies default encoding, defaults to 'UTF-8'

\item[\code{curlOpts}] a named list or CURLOptions object identifying the curl options for the handle. Type \code{listCurlOptions()} for all Curl options available.

\item[\code{postFUN}] function saved in WebSource object and called to retrieve full text content from feed urls

\item[\code{retrieveFeedURL}] logical; Specify if feedurls should be downloaded first.

\item[\code{...}] additional parameters passed to \code{WebSource} object/structure
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
\inputencoding{utf8}
\HeaderA{YahooFinanceSource}{Get feed data from Yahoo! Finance.}{YahooFinanceSource}
\aliasA{readYahoo}{YahooFinanceSource}{readYahoo}
%
\begin{Description}\relax
Yahoo! Finance is a popular site which provides financial news and information. It is a large source
for historical price data as well as financial news. Using the typical Yahoo! Finance ticker
news items can easily be retrieved. However, the maximum number of items is 20.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
YahooFinanceSource(query, params = list(s = query, region = "US", lang =
  "en-US"), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] ticker symbols of companies to be searched for, see \url{http://finance.yahoo.com/lookup}.

\item[\code{params,}] additional query parameters, see \url{http://developer.yahoo.com/rss/}

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebXMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(YahooFinanceSource("MSFT"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{YahooInplaySource}{Get News from Yahoo Inplay.}{YahooInplaySource}
\aliasA{readYahooInplay}{YahooInplaySource}{readYahooInplay}
%
\begin{Description}\relax
Yahoo Inplay lists a range of company news provided by Briefing.com. Since Yahoo Inplay
does not provide a structured XML news feed, content is parsed directly from the HTML page.
Therefore, no further Source parameters can be specified. The number of feed items per
request can vary substantially.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
YahooInplaySource(...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebHTMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(YahooInplaySource())

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{yahoonews}{WebCorpus retrieved from Yahoo! News for the search term "Microsoft" through the YahooNewsSource. Length of retrieved corpus is 20.}{yahoonews}
\keyword{data}{yahoonews}
%
\begin{Description}\relax
WebCorpus retrieved from Yahoo! News for the search term "Microsoft"
through the YahooNewsSource. Length of retrieved corpus is 20.
\end{Description}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{Examples}
\begin{ExampleCode}
#Data set has been generated as follows:
## Not run: 
yahoonews <- WebCorpus(YahooNewsSource("Microsoft"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\inputencoding{utf8}
\HeaderA{YahooNewsSource}{Get news data from Yahoo! News (\url{https://news.search.yahoo.com/search/}).}{YahooNewsSource}
\aliasA{readYahooHTML}{YahooNewsSource}{readYahooHTML}
%
\begin{Description}\relax
Currently, only a maximum of 10 items can be retrieved.
\end{Description}
%
\begin{Usage}
\begin{verbatim}
YahooNewsSource(query, params = list(p = query), ...)
\end{verbatim}
\end{Usage}
%
\begin{Arguments}
\begin{ldescription}
\item[\code{query}] words to be searched in Yahoo News, multiple words must be separated by '+'

\item[\code{params,}] additional query parameters, see \url{http://developer.yahoo.com/rss/}

\item[\code{...}] additional parameters to \code{\LinkA{WebSource}{WebSource}}
\end{ldescription}
\end{Arguments}
%
\begin{Value}
WebXMLSource
\end{Value}
%
\begin{Author}\relax
Mario Annau
\end{Author}
%
\begin{SeeAlso}\relax
\code{\LinkA{WebSource}{WebSource}}
\end{SeeAlso}
%
\begin{Examples}
\begin{ExampleCode}
## Not run: 
corpus <- Corpus(YahooNewsSource("Microsoft"))

## End(Not run)
\end{ExampleCode}
\end{Examples}
\printindex{}
\end{document}
