Python ? 我有足够的声誉，我可以在不需要你接受的情况下进行编辑。我可以在数据库中插入链接结果的完整链接。问题是，程序第一次执行时，在所有程序终止之前，行不应该是1。然后，假设我在5分钟后再次运行程序，当新数据与已提取的数据冲突时，将提取更多的数据并停止程序（通过
Python ? 我有足够的声誉，我可以在不需要你接受的情况下进行编辑。我可以在数据库中插入链接结果的完整链接。问题是，程序第一次执行时，在所有程序终止之前，行不应该是1。然后，假设我在5分钟后再次运行程序，当新数据与已提取的数据冲突时，将提取更多的数据并停止程序（通过,python,mysql,sql,database,shell,Python,Mysql,Sql,Database,Shell,? 我有足够的声誉，我可以在不需要你接受的情况下进行编辑。我可以在数据库中插入链接结果的完整链接。问题是，程序第一次执行时，在所有程序终止之前，行不应该是1。然后，假设我在5分钟后再次运行程序，当新数据与已提取的数据冲突时，将提取更多的数据并停止程序（通过匹配链接结果的结果，但在我的情况下，在其中一个程序中（除URL外，所有程序都相同）即使数据库为空，也不会执行。 db= MySQLdb.connect("localhost","root","ahmed","practice")
? 我有足够的声誉，我可以在不需要你接受的情况下进行编辑。我可以在数据库中插入
链接结果的完整链接。问题是，程序第一次执行时，在所有程序终止之前，行不应该是1。然后，假设我在5分钟后再次运行程序，当新数据与已提取的数据冲突时，将提取更多的数据并停止程序（通过匹配链接结果的结果，但在我的情况下，在其中一个程序中（除URL外，所有程序都相同）即使数据库为空，也不会执行。
db= MySQLdb.connect("localhost","root","ahmed","practice")
cursor=db.cursor()
#checking phase to stop scrapping
sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
print rows
rows = cursor.execute(sql,(link_result))
print rows
if rows>=1:
print "Already present - The program is terminating"
sys.exit()
else:
query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))
db.commit()
cursor.close()
db.close()
all.sh
python python1.py &
python python2.py &
python python3.py &
python python4.py &
python python5.py &
#!/usr/bin/python
import urllib
from bs4 import BeautifulSoup
import MySQLdb
import re
import pdb
import sys
def getting_urls_of_all_pages():
i=1
while i<=40: #40 is the total number of main pages
url_rent_flat='http://dubai.dubizzle.com/property-for-rent/residential/apartmentflat/?page='+str(i) #url of the main page (iterating to 40)
link=[]
htmlfile=urllib.urlopen(url_rent_flat).read()
soup=BeautifulSoup(htmlfile)
link=soup.find_all('a',xtclib=re.compile("listing_list_\d+_title_link"),href=True) #stores all the links (25) links of the page
"""
Part 2: passing each property url to process for data extraction
"""
for a in link:
every_property_in_a_page_data_extraction(a['href'])
i+=1
def every_property_in_a_page_data_extraction(url):
title_result=""
price_result=""
bedroom_result=""
agencyfee_result=""
bathroom_result=""
size_result=""
propertyref_result=""
furnished_result=""
rent_is_paid_result=""
building_result=""
Amenities_result=""
tradename_result=""
licencenum_result=""
reraid_result=""
phone_result=""
link_result=""
Zoned_for_result=""
Freehold_result=""
Pricepersq_result=""
Type_result="Apartment"
Sale_Rent_result="Rent"
rows=0
"""
Part1: Extracting data
"""
htmlfile=urllib.urlopen(url).read()
soup=BeautifulSoup(htmlfile)
"""
Part2: Extracting the components
"""
# Sale/Rent
print "Sale/Rent: ", Sale_Rent_result
# Type of property
print "Type of property: ", Type_result
#title
try:
title= soup.find('span',{'id':'listing-title-wrap'})
title_result= str(title.get_text().strip().encode("utf-8"))
print "Title: ",title_result
except StandardError as e:
title_result="Error was {0}".format(e)
print title_result
#price
try:
price = soup.find('span',{'id':'actualprice'})
price_result= str(price.get_text())
print "Price: ",price_result
except StandardError as e:
price_result="Error was {0}".format(e)
print price_result
#Agency Fee, Bedroom, Bathroom, Size
spans_ABBS= []
for a in soup.select(".important-fields li span"):
spans_ABBS.append(a.text.strip())
strongs_ABBS=[]
for a in soup.select(".important-fields li strong"):
strongs_ABBS.append(a.text.strip())
for name, value in zip(spans_ABBS, strongs_ABBS):
if name=="Agency Fees:":
try:
agencyfee_result= str(value)
print "Agency Fees: ", agencyfee_result
except StandardError as e:
agencyfee_result="Error was {0}".format(e)
print agencyfee_result
elif name=="Bedrooms:":
try:
bedroom_result= str(value)
print "Number of Bedrooms: ",bedroom_result
except StandardError as e:
bedroom_result="Error was {0}".format(e)
print bedroom_result
elif name=="Bathrooms:":
try:
bathroom_result= str(value)
print "Number of Bathrooms: ", bathroom_result
except StandardError as e:
bathroom_result="Error was {0}".format(e)
print bathroom_result
elif name=="Size:":
try:
size_result= str(value)
print "Size of the property: ",size_result
except StandardError as e:
size_result="Error was {0}".format(e)
print size_result
elif name=="Zoned For:":
try:
Zoned_for_result= str(value)
print "Zoned For:",Zoned_for_result
except StandardError as e:
Zoned_for_result="Error was {0}".format(e)
print Zoned_for_result
elif name=="Freehold:":
try:
Freehold_result= str(value)
print "Freehold: ",Freehold_result
except StandardError as e:
Freehold_result="Error was {0}".format(e)
print Freehold_result
elif name=="Price / SqFt:":
try:
Pricepersq_result= str(value)
print "Price Per Sqft: ",Pricepersq_result
except StandardError as e:
Pricepersq_result="Error was {0}".format(e)
print Pricepersq_result
#Property Reference, Furnished, Listed By, Rent Is Paid, Building, Amenities:
spans_others=[]
for a in soup.select("#listing-details-list li span"):
spans_others.append(a.text.strip())
strongs_others=[]
for a in soup.select("#listing-details-list li strong"):
strongs_others.append(a.text.strip())
for name, value in zip(spans_others, strongs_others):
if name=="Listed by:":
break
elif name=="Property Reference:":
try:
propertyref_result=str(value.strip())
print "Property reference in Dubizel: ",propertyref_result
except StandardError as e:
propertyref_result="Error was {0}".format(e)
print propertyref_result
elif name=="Furnished:":
try:
furnished_result=str(value.strip())
print "Furnished status: ",furnished_result
except StandardError as e:
furnished_result="Error was {0}".format(e)
print furnished_result
elif name=="Rent Is Paid:":
try:
rent_is_paid_result=str(value.strip())
print "Rent payment: ",rent_is_paid_result
except StandardError as e:
rent_is_paid_result="Error was {0}".format(e)
print rent_is_paid_result
elif name=="Building:":
try:
building_result=str(value.strip())
print "Building info: ",building_result
except StandardError as e:
building_result="Error was {0}".format(e)
print building_result
elif name=="Amenities:":
try:
for a in value.split(","):
Amenities_result+=a.strip()+","
print Amenities_result
except StandardError as e:
Amenities_result="Error was {0}".format(e)
print Amenities_result
#Agents info --> TTrade Name, DED Licence Number, RERA Registration Number
spans_broker=[]
for a in soup.select("#broker-details li span"):
spans_broker.append(a.text.strip())
strongs_broker=[]
for a in soup.select("#broker-details li strong"):
strongs_broker.append(a.text.strip())
for name, value in zip(spans_broker, strongs_broker):
if name=="Trade Name:":
try:
tradename_result=str(value.strip())
print "Trade name: ",tradename_result
except StandardError as e:
tradename_result="Error was {0}".format(e)
print tradename_result
elif name=="DED Licence Number:":
try:
licencenum_result=str(value.strip())
print "Licence #: ",licencenum_result
except StandardError as e:
licencenum_result="Error was {0}".format(e)
print licencenum_result
elif name=="RERA Registration Number:":
try:
reraid_result=str(value.strip())
print "RERA ID #: ",reraid_result
except StandardError as e:
reraid_result="Error was {0}".format(e)
print reraid_result
# phone num
try:
phone=soup.find_all("div", "phone-content")
for a in phone:
phone_result= str(a.get_text().strip().encode("utf-8"))
print "Phone information:", phone_result
except StandardError as e:
phone_result="Error was {0}".format(e)
print phone_result
#link
try:
link = soup.find('input',{'id':'short-link-input'})
link_result= str(link.get('value'))
print "Short Reference link: ", link_result
except StandardError as e:
link_result="Error was {0}".format(e)
print link_result
"""
Connecting to Database and putting data into in
"""
db= MySQLdb.connect("localhost","root","ahmed","practice")
cursor=db.cursor()
#checking phase to stop scrapping
sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
print rows
rows = cursor.execute(sql,(link_result))
print rows
if rows>=1:
print "Already present - The program is terminating"
sys.exit()
else:
query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))
db.commit()
cursor.close()
db.close()
#-----------------------------------------------------------
getting_urls_of_all_pages()
rows = cursor.execute(sql,(link_result))
rows = cursor.execute(sql, link_result)
rows = cursor.execute(sql, (link_result,))