import urllib2
from bs4 import BeautifulSoup
import requests
import os
import sys
from requests.exceptions import HTTPError
sys.path.append("../../pypoj")

from requests.exceptions import HTTPError
from datainitializer.AppPipeline import AppPipeline
from datainitializer.AppItem import AppItem
from config.constant import Values


db_pipeline = AppPipeline(Values.fdroid_28_9_2018_path)
db_pipeline.initialize()



#parse fdroid dowload page and download apk
def parse_fdroid_dowload_page(apk_pkg_name):
	apkname = apk_pkg_name[0]
	pkgname = apk_pkg_name[1]
	if apk_pkg_name[2]:
		return

	decoded_apks_record = Values.log_root+'/decoded_apks.txt'
	recordFile = open(decoded_apks_record, 'a')
	recordFile.write('%s\n' % (pname[0]))
	recordFile.close()

	page_url = 'https://f-droid.org/en/packages/'+pkgname
	try:
		# query the website and return the html to the variable 'page'
		page = urllib2.urlopen(page_url)

		# parse the html using beautiful soup and store in variable 'soup'
		soup = BeautifulSoup(page, "lxml")
		app_name = None
		app_summary = None
		app_description = None
		if soup.find('h3', attrs={'class':'package-name'}) is not None:
			app_name = soup.find('h3', attrs={'class':'package-name'}).get_text().strip()
		if soup.find('div', attrs={'class':'package-summary'}) is not None:
			app_summary = soup.find('div', attrs={'class':'package-summary'}).get_text().strip()
		
		if soup.find('div', attrs={'class':'package-description'}) is not None:
			app_description = soup.find('div', attrs={'class':'package-description'}).get_text().strip()

		print app_name
		print app_summary
		print app_description
		print '\n'
		db_pipeline.update_name_summary_desc(app_name,app_summary,app_description, apkname)
	except urllib2.HTTPError, e:
		with open(Values.log_root+"/except.txt", "a") as myfile:
			myfile.write(str(e.code) + ' Could not download page ' + page_url + "\n")
    		myfile.close()


pkgnames = db_pipeline.get_all_apps()
for pname in pkgnames:
	parse_fdroid_dowload_page(pname)




db_pipeline.finalize()

