Classes
class	AllFeedsCollector

class	Article

class	DataHandler

class	Feed

class	FeedCollector

class	HtmlFile

class	MainHandler

class	OAuthHandler

class	Section

class	SetDefaultFeeds

class	SetTestFeeds

class	TopHandler

class	UpdateHtml

class	UploadFeed

class	UserData

class	UserLoginHandler

Functions
	findSectionByTitle (title)

	collectFeed (feed, data, continuation=None)

	collectArticle (feed, data)

	unescape (html)

	getFeedIcon (url)

	findImage (text)

	findImgTag (text, extensions)

	findVideoTag (text)

	makeThumbnail (text)

	generateThumbnail (url)

	main ()

	Main (root_directory)

	GetAllFilepaths (root_directory)

	ReportWarning (text)

	ReportError (text)

	ReadFileIntoString (filepath)

	WriteStringToFile (string, filepath)

	ConfirmAllowedCopyrightHolder (holder)

Variables
tuple	THUMB_SIZE = (57, 57)

str	READER_API = 'http://www.google.com/reader/api/0'

int	MAX_SECTIONS = 5

int	MAX_ARTICLES = 20

int	UPDATE_COUNT = 4

int	INITIAL_COUNT = 40

int	SNIPPET_SIZE = 180

list	ALLOWED_COPYRIGHT_HOLDERS

Detailed Description

Copyright 2011 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.

Function Documentation

◆ collectArticle()

main.collectArticle	(	feed,
		data
	)

Reads an article from the given JSON object and populates the datastore with
it.

Definition at line 614 of file main.py.

def collectArticle(feed, data):
    '''
  Reads an article from the given JSON object and populates the datastore with
  it.
  '''
    if not 'title' in data:
        # Skip this articles without titles
        return True
 
    articleId = data['id']
    article = Article.get_or_insert(articleId)
    # TODO(jimhug): This aborts too early - at lease for one adafruit case.
    if article.date == data['published']:
        logging.info('found existing, aborting: %r, %r' %
                     (articleId, article.date))
        return False
 
    if data.has_key('content'):
        content = data['content']['content']
    elif data.has_key('summary'):
        content = data['summary']['content']
    else:
        content = ''
    #TODO(jimhug): better summary?
    article.content = content
    article.date = data['published']
    article.title = unescape(data['title'])
    article.snippet = unescape(strip_tags(content)[:SNIPPET_SIZE])
 
    article.feed = feed
 
    # TODO(jimhug): make this canonical so UX can change for this state
    article.author = data.get('author', 'anonymous')
 
    article.ensureThumbnail()
 
    article.srcurl = ''
    if data.has_key('alternate'):
        for alt in data['alternate']:
            if alt.has_key('href'):
                article.srcurl = alt['href']
    return True
 
 

◆ collectFeed()

main.collectFeed	(	feed,
		data,
		continuation = `None`
	)

Reads a feed from the given JSON object and populates the given feed object
in the datastore with its data.

Definition at line 585 of file main.py.

def collectFeed(feed, data, continuation=None):
    '''
  Reads a feed from the given JSON object and populates the given feed object
  in the datastore with its data.
  '''
    if continuation is None:
        if 'alternate' in data:
            feed.iconUrl = getFeedIcon(data['alternate'][0]['href'])
        feed.title = data['title']
        feed.lastUpdated = data['updated']
 
    articles = data['items']
    logging.info('%d new articles for %s' % (len(articles), feed.title))
 
    for articleData in articles:
        if not collectArticle(feed, articleData):
            feed.put()
            return False
 
    if len(articles) > 0 and data.has_key('continuation'):
        logging.info('would have looked for more articles')
        # TODO(jimhug): Enable this continuation check when more robust
        #self.fetchn(feed, feedId, data['continuation'])
 
    feed.ensureEncodedFeed(force=True)
    feed.put()
    return True
 
 

◆ ConfirmAllowedCopyrightHolder()

main.ConfirmAllowedCopyrightHolder ( holder )

Returns True if this is one of our allowed copyright holders.

@param holder copyright holder as a string

Definition at line 108 of file main.py.

def ConfirmAllowedCopyrightHolder(holder):
    """Returns True if this is one of our allowed copyright holders.
 
    @param holder copyright holder as a string
    """
    return holder in ALLOWED_COPYRIGHT_HOLDERS
 
 

◆ findImage()

main.findImage ( text )

Definition at line 671 of file main.py.

def findImage(text):
    img = findImgTag(text, 'jpg|jpeg|png')
    if img is not None:
        return img
 
    img = findVideoTag(text)
    if img is not None:
        return img
 
    img = findImgTag(text, 'gif')
    return img
 
 

◆ findImgTag()

main.findImgTag	(	text,
		extensions
	)

Definition at line 684 of file main.py.

def findImgTag(text, extensions):
    m = re.search(r'src="(http://\S+\.(%s))(\?.*)?"' % extensions, text)
    if m is None:
        return None
    return m.group(1)
 
 

◆ findSectionByTitle()

main.findSectionByTitle ( title )

Definition at line 578 of file main.py.

def findSectionByTitle(title):
    for section in Section.all():
        if section.fixedTitle() == title:
            return section
    return None
 
 

◆ findVideoTag()

main.findVideoTag ( text )

Definition at line 691 of file main.py.

def findVideoTag(text):
    # TODO(jimhug): Add other videos beyond youtube.
    m = re.search(r'src="http://www.youtube.com/(\S+)/(\S+)[/|"]', text)
    if m is None:
        return None
 
    return 'http://img.youtube.com/vi/%s/0.jpg' % m.group(2)
 
 

◆ generateThumbnail()

main.generateThumbnail ( url )

Definition at line 712 of file main.py.

def generateThumbnail(url):
    logging.info('generating thumbnail: %s' % url)
    thumbWidth, thumbHeight = THUMB_SIZE
 
    result = urlfetch.fetch(url)
    img = images.Image(result.content)
 
    w, h = img.width, img.height
 
    aspect = float(w) / h
    thumbAspect = float(thumbWidth) / thumbHeight
 
    if aspect > thumbAspect:
        # Too wide, so crop on the sides.
        normalizedCrop = (w - h * thumbAspect) / (2.0 * w)
        img.crop(normalizedCrop, 0., 1. - normalizedCrop, 1.)
    elif aspect < thumbAspect:
        # Too tall, so crop out the bottom.
        normalizedCrop = (h - w / thumbAspect) / h
        img.crop(0., 0., 1., 1. - normalizedCrop)
 
    img.resize(thumbWidth, thumbHeight)
 
    # Chose JPEG encoding because informal experiments showed it generated
    # the best size to quality ratio for thumbnail images.
    nimg = img.execute_transforms(output_encoding=images.JPEG)
    logging.info('  finished thumbnail: %s' % url)
 
    return nimg
 
 

◆ GetAllFilepaths()

main.GetAllFilepaths ( root_directory )

Return a list of all files (absolute path for each one) within a tree.

@param root_directory root directory within which to find all files

Definition at line 68 of file main.py.

def GetAllFilepaths(root_directory):
    """Return a list of all files (absolute path for each one) within a tree.
 
    @param root_directory root directory within which to find all files
    """
    path_list = []
    for dirpath, _, filenames in os.walk(root_directory):
        for filename in filenames:
            path_list.append(os.path.abspath(os.path.join(dirpath, filename)))
    return path_list
 
 

◆ getFeedIcon()

main.getFeedIcon ( url )

Definition at line 666 of file main.py.

def getFeedIcon(url):
    url = urlparse.urlparse(url).netloc
    return 'http://s2.googleusercontent.com/s2/favicons?domain=%s&alt=feed' % url
 
 

◆ main()

main.main ( )

Definition at line 758 of file main.py.

def main():
    application = webapp.WSGIApplication(
        [
            ('/data/(.*)', DataHandler),
 
            # This is called periodically from cron.yaml.
            ('/update/allFeeds', AllFeedsCollector),
            ('/update/feed', FeedCollector),
            ('/update/user', UserLoginHandler),
            ('/update/defaultFeeds', SetDefaultFeeds),
            ('/update/testFeeds', SetTestFeeds),
            ('/update/html', UpdateHtml),
            ('/update/upload', UploadFeed),
            ('/oauth2callback', OAuthHandler),
            ('/', TopHandler),
            ('/(.*)', MainHandler),
        ],
        debug=True)
    webapp.util.run_wsgi_app(application)
 
 

◆ Main()

main.Main ( root_directory )

Run everything.

@param root_directory root directory within which to modify all files

Definition at line 33 of file main.py.

def Main(root_directory):
    """Run everything.
 
    @param root_directory root directory within which to modify all files
    """
    filepaths = GetAllFilepaths(root_directory)
    for filepath in filepaths:
        parser = fileparser.CreateParser(filepath)
        if not parser:
            ReportWarning('cannot find a parser for file %s, skipping...' %
                          filepath)
            continue
        old_file_contents = ReadFileIntoString(filepath)
        comment_blocks = parser.FindAllCommentBlocks(old_file_contents)
        if not comment_blocks:
            ReportWarning('cannot find any comment blocks in file %s' %
                          filepath)
        old_copyright_block = parser.FindCopyrightBlock(comment_blocks)
        if not old_copyright_block:
            ReportWarning('cannot find copyright block in file %s' % filepath)
        (year, holder) = parser.GetCopyrightBlockAttributes(old_copyright_block)
        if holder and not ConfirmAllowedCopyrightHolder(holder):
            ReportWarning(
                'unrecognized copyright holder "%s" in file %s, skipping...' % (
                    holder, filepath))
            continue
        new_copyright_block = parser.CreateCopyrightBlock(year, holder)
        if old_copyright_block:
            new_file_contents = old_file_contents.replace(
                old_copyright_block, new_copyright_block, 1)
        else:
            new_file_contents = new_copyright_block + old_file_contents
        WriteStringToFile(new_file_contents, filepath)
 
 

◆ makeThumbnail()

main.makeThumbnail ( text )

Definition at line 700 of file main.py.

def makeThumbnail(text):
    url = None
    try:
        url = findImage(text)
        if url is None:
            return None
        return generateThumbnail(url)
    except:
        logging.info('error decoding: %s' % (url or text))
        return None
 
 

◆ ReadFileIntoString()

main.ReadFileIntoString ( filepath )

Returns the full contents of this file as a string.

Definition at line 92 of file main.py.

def ReadFileIntoString(filepath):
    """Returns the full contents of this file as a string.
    """
    with open(filepath, 'r') as file_handle:
        contents = file_handle.read()
    return contents
 
 

◆ ReportError()

main.ReportError ( text )

Report an error and raise an exception.

Definition at line 86 of file main.py.

def ReportError(text):
    """Report an error and raise an exception.
    """
    raise IOError(text)
 
 

◆ ReportWarning()

main.ReportWarning ( text )

Report a warning, but continue.

Definition at line 80 of file main.py.

def ReportWarning(text):
    """Report a warning, but continue.
    """
    print('warning: %s' % text)
 
 

◆ unescape()

main.unescape ( html )

Definition at line 658 of file main.py.

def unescape(html):
    "Inverse of Django's utils.html.escape function"
    if not isinstance(html, basestring):
        html = str(html)
    html = html.replace('&#39;', "'").replace('&quot;', '"')
    return html.replace('&gt;', '>').replace('&lt;', '<').replace('&amp;', '&')
 
 

◆ WriteStringToFile()

main.WriteStringToFile	(	string,
		filepath
	)

Writes this string out to filepath, replacing the file if it already
exists.

Definition at line 100 of file main.py.

def WriteStringToFile(string, filepath):
    """Writes this string out to filepath, replacing the file if it already
    exists.
    """
    with open(filepath, 'w') as file_handle:
        file_handle.write(string)
 
 

Variable Documentation

◆ ALLOWED_COPYRIGHT_HOLDERS

list main.ALLOWED_COPYRIGHT_HOLDERS

Initial value:

=  [
    'Google Inc.',
    'Skia',
    'The Android Open Source Project',
]

Definition at line 27 of file main.py.

◆ INITIAL_COUNT

int main.INITIAL_COUNT = 40

Definition at line 538 of file main.py.

◆ MAX_ARTICLES

int main.MAX_ARTICLES = 20

Definition at line 28 of file main.py.

◆ MAX_SECTIONS

int main.MAX_SECTIONS = 5

Definition at line 27 of file main.py.

◆ READER_API

str main.READER_API = 'http://www.google.com/reader/api/0'

Definition at line 25 of file main.py.

◆ SNIPPET_SIZE

int main.SNIPPET_SIZE = 180

Definition at line 539 of file main.py.

◆ THUMB_SIZE

tuple main.THUMB_SIZE = (57, 57)

Definition at line 24 of file main.py.

◆ UPDATE_COUNT

int main.UPDATE_COUNT = 4

Definition at line 537 of file main.py.

Classes

Functions

Variables