import urllib2
from bs4 import BeautifulSoup
import requests
import os
import sys
import argparse
sys.path.append("../../pypoj")
from requests.exceptions import HTTPError
from datainitializer.ApiPipeline import ApiPipeline
from datainitializer.ApiItem import ApiItem
from config.constant import Values

db_pipeline = None
# logFile = open('log.txt', 'a')

def get_descr(descrs):
	des = ''
	for tag in descrs:
		des+=tag.get_text().strip().replace('\n', ' ')
	return des.replace('\n', ' ')

def get_perms(perm):
	permstr = None
	des = perm.get_text().strip().replace('\n', ' ')
	if des.startswith('Requires the'):
		for tag in perm.find_all('a'):
			if permstr is None:
				permstr = tag.get_text().strip().replace('\n', ' ')
			else:
				permstr += '++'+tag.get_text().strip().replace('\n', ' ')
	return permstr

def parse_api_doc_page(sensitive_api):
	apinfos = sensitive_api.split('-')
	if len(apinfos) < 5:
		with open("fail_record.txt", "a") as myfile:
			myfile.write(sensitive_api + "\n")
    		myfile.close()
		return

	clsname = apinfos[0]
	mtdname = apinfos[1]
	restriction_name = apinfos[2]
	specifier = None if apinfos[3]==mtdname else apinfos[3]
	mtdsig = apinfos[4].replace('$','.')
	
	if db_pipeline.get_api_by_sig_restrname_specifier(mtdsig, restriction_name, specifier) is not None:
		return

	page_url = 'https://developer.android.com/reference/'+clsname.replace('.','/')
	print page_url
	print '------',mtdsig
	# logFile.write('%s\n' % (page_url))
	# logFile.write('-----%s\n' % (mtdsig))

	summary = None
	descr = None
	apilevel = None
	sig = None
	perms = None
	try:
		# query the website and return the html to the variable 'page'
		page = urllib2.urlopen(page_url)

		# parse the html using beautiful soup and store in variable 'soup'
		soup = BeautifulSoup(page, "lxml")

		for tag in soup.find_all('h3', attrs={'class':'api-name'}):
			name = tag.get_text().strip().replace('\n', ' ')
			if name == mtdname and 'id' in tag.attrs:
				# sig = tag['id'].strip().replace('%20',' ')
				sig=tag.find_next_sibling('pre', attrs={'class':'api-signature no-pretty-print'}).get_text().strip().replace('\n', ' ').replace('                  ', ' ')
				# if sig in mtdsig:
				print '++++++',sig.encode('utf-8')
				# logFile.write('++++++%s\n' % (sig.encode('utf-8')))

				apilevel=tag.find_next_sibling('div', attrs={'class':'api-level'}).find('a').get_text().strip().replace('\n', ' ')

				psibs = tag.find_next_siblings('p')
				summary = psibs[0].get_text().strip().replace('\n', ' ')
				if len(psibs) > 1:
					descr = get_descr(psibs[1:])
				perms = get_perms(psibs[-1])
				api_item = ApiItem(clsname,mtdname,restriction_name,specifier,sig,apilevel,summary,descr,perms)
				# print api_item
				db_pipeline.insert_into_DB(api_item)
					# print descr
	except urllib2.HTTPError, e:
		with open("except.txt", "a") as myfile:
			myfile.write(str(e.code) + ' Could not download page ' + page_url + "\n")
    		myfile.close()
	if apilevel is None:
		api_item = ApiItem(clsname,mtdname,restriction_name,specifier,mtdsig,apilevel,summary,descr,perms)
		db_pipeline.insert_into_DB(api_item)

		with open("no_doc.txt", "a") as myfile:
			myfile.write(sensitive_api + "\n")
    		myfile.close()









if __name__ == "__main__":
	

	#process arguments
	parser = argparse.ArgumentParser(prog = 'python sensitive_api_info_crawl.py', description = '')
	# parser.add_argument('-result', nargs = 1, help = 'specify the directory of the file saving results of the analysis', default = ['/home/jliu/Research/tools/results/'])
	# parser.add_argument('-find', nargs = 1, default = ['exec'], help = 'specify string being found')
	args = parser.parse_args()


	# apiinfo_db = os.path.join(result_dir, 'sensitive_api_infos.db')
	db_pipeline = ApiPipeline(Values.sensitive_api_infos_path)
	db_pipeline.initialize()
	

	sensitive_apis = set(line.strip() for line in open(Values.memhook_path))
	i=1
	for api in sensitive_apis:
		parse_api_doc_page(api)
		print '-+-+-+-',i,api
		i+=1
	
	
	db_pipeline.finalize()
	# logFile.close()

