import csv import os import time import requests import json import re class Zillow: def __init__(self): self.baseurl = "https://www.zillow.com" self.proxies = { "http": "http://127.0.0.1:7890", "https": "http://127.0.0.1:7890", } self.cookies = { 'zguid': '24|%24e835e7de-1e03-40a2-9e0b-ea6557fce975', 'zgsession': '1|5a92d272-1f0a-4eab-aad9-0b01bfc4e12e', 'pxcts': '4efbe577-f671-11ef-944c-1da0cdd7af45', '_pxvid': '4efbd04b-f671-11ef-944a-e6989223c886', 'zjs_anonymous_id': '%22e835e7de-1e03-40a2-9e0b-ea6557fce975%22', 'zg_anonymous_id': '%224b8a74fb-12d3-4dd5-b453-91d2f059cc5c%22', '_ga': 'GA1.2.1116450831.1740815202', '_gid': 'GA1.2.1210142068.1740815202', '_gcl_au': '1.1.1930799100.1740815207', '_scid': '3cxthBM96c4ODxO1v0Al3ufRKQFwF0wr', 'DoubleClickSession': 'true', '_ScCbts': '%5B%5D', '_pin_unauth': 'dWlkPU4yRmtNVGhtTmpVdE5tWmtaUzAwWXpnNUxUbGtPRGt0WkRrek1UTm1OREUyTTJJdw', '_tt_enable_cookie': '1', '_ttp': '01JN8AR8CC889QQ2N91WJNGVRG_.tt.1', '_fbp': 'fb.1.1740815213095.801729201273859650', '_sctr': '1%7C1740758400000', '_clck': '1gi6cgn%7C2%7Cftu%7C0%7C1886', '_lr_env_src_ats': 'false', 'g_state': '{"i_l":0}', 'loginmemento': '1|38bd958cf99c0efa6fcdf2d8b4ba656e18214f94b1d0b8482177e5821d485cb5', 'userid': 'X|3|516ce03fb1857a73%7C9%7CtUJB3zsjokPFJzXVcdK9kYKiV6BT9Hom', 'zjs_user_id': '%22X1-ZUqbce6dl8y0w9_1oeg0%22', '_derived_epik': 'dj0yJnU9YzhHUUI3UjgwVTYyektsUXlURWZ5UnNPZGpPTDBwUi0mbj01MkNxOF9nSm1udHEwczFUVGoyM0p3Jm09NCZ0PUFBQUFBR2ZDeDJRJnJtPTQmcnQ9QUFBQUFHZkN4MlEmc3A9Mg', 'tfpsi': 'cf8acdc2-7676-40ea-810c-a9d9613cd3ac', 'ZILLOW_SID': '1|AAAAAVVbFRIBVVsVEuEBjRdhLdPgArRt9zTF9A9GUy9n6qhArfRCvbRpWcUyvOX17mDKQCRGzog4qedbe0aFqFnGaQzFa2AVHA', 'JSESSIONID': '2AB46AF163E850A6A93A55EEAFBFA961', '_px3': '2b2ceb1e9473516a5889d6b089882f2df95563f843864b748551a222e3a93bd6:Re6tLyXWJyuSifQZrCVdgU+HEPF8Ih1TmBAyFOt2qBZwjYXKAdScUmpmzzTUHclqZwD1vCAkP6Pow8uPxHGEBw==:1000:8rliApJL6+buJDD0QJN5jVM3zTksVkovJZ9PzmBfy8r178UA1kmnQ3o8GCOS0s/wmN0v5TATeetgtPpn8ZgJH/nA3DeEY9emAKW8bwwCOzP2xxF8rOEN4IYvM48vCdaEhFnLMTOWUR5ZhrMtnuinXvnFPpcsoTdiVsjPiPUunfMbpjRVUSEeRGfrO2BvQBnqh1bPhqq4CXReAdRoEbUdIoO+jZ250vTxbnHyWHDnnsk=', '_rdt_uuid': '1740815209620.81e8d67a-5a8f-489d-844a-6eb51bea084d', '_scid_r': '7ExthBM96c4ODxO1v0Al3ufRKQFwF0wrejgyEw', 'search': '6|1743432649336%7Crect%3D40.945233052704474%2C-73.83509060791016%2C40.845317795985295%2C-73.98271939208985%26rid%3D17182%26disp%3Dmap%26mdm%3Dauto%26p%3D1%26listPriceActive%3D1%26fs%3D1%26fr%3D0%26mmm%3D0%26rs%3D0%26singlestory%3D0%26housing-connector%3D0%26parking-spots%3Dnull-%26abo%3D0%26garage%3D0%26pool%3D0%26ac%3D0%26waterfront%3D0%26finished%3D0%26unfinished%3D0%26cityview%3D0%26mountainview%3D0%26parkview%3D0%26waterview%3D0%26hoadata%3D1%26zillow-owned%3D0%263dhome%3D0%26showcase%3D0%26featuredMultiFamilyBuilding%3D0%26onlyRentalStudentHousingType%3D0%26onlyRentalIncomeRestrictedHousingType%3D0%26onlyRentalMilitaryHousingType%3D0%26onlyRentalDisabledHousingType%3D0%26onlyRentalSeniorHousingType%3D0%26commuteMode%3Ddriving%26commuteTimeOfDay%3Dnow%09%0917182%09%7B%22isList%22%3Atrue%2C%22isMap%22%3Atrue%7D%09%09%09%09%09', '_clsk': 'uwbs56%7C1740840661345%7C2%7C0%7Cr.clarity.ms%2Fcollect', 'AWSALB': 'SVlf4eVEwg9mRsnpIwRyN+xS9wd8r/tNawixFSnBEf7jkmDoA1qS5ygg/3YxMalClDa1hkwxgRsZsha0aX46GGoXENyABk7jsy4YM0+YnjOiQ4CnF/ZFkf377PDR', 'AWSALBCORS': 'SVlf4eVEwg9mRsnpIwRyN+xS9wd8r/tNawixFSnBEf7jkmDoA1qS5ygg/3YxMalClDa1hkwxgRsZsha0aX46GGoXENyABk7jsy4YM0+YnjOiQ4CnF/ZFkf377PDR', '_uetsid': '566607f0f67111ef9e8f315a86932718', '_uetvid': '5665f370f67111efae377d2348e45d1f', } self.csvdata = [ [ 'zpid', 'price', # 价格 'yearBuilt', # 始建 'streetAddress', # 地址 'bedrooms', # 卧室数量 'bathrooms', # 浴室数量 'responsivePhotos', # 照片 'timeOnZillow', # 展示天数 'pageViewCount', # 页面浏览量 'favoriteCount', # 收藏数量 'phrases', # 标签 'description', # 描述 ], ] if not os.path.exists('data.csv'): with open('data.csv', 'w', newline='', encoding='utf-8') as f: writer = csv.writer(f) writer.writerow(self.csvdata[0]) # 写入标题行 # self.update_cookies_ZILLOW_SID() # 更新cookies def get_JSESSIONID(self): cookies = self.cookies headers = { 'accept': '*/*', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'pragma': 'no-cache', 'priority': 'u=1, i', 'referer': 'https://www.zillow.com/homedetails/4705-Henry-Hudson-Pkwy-APT-6B-Bronx-NY-10471/244446711_zpid/', 'sec-ch-ua': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36' } params = { 'zpid': '244446711', } response = requests.get( 'https://www.zillow.com/ajax/homedetail/MarkPropertyViewed.htm', params=params, cookies=cookies, headers=headers, ) JSESSIONID = response.headers.get('x-requested-session') print(JSESSIONID) self.cookies['JSESSIONID'] = JSESSIONID cookies = response.headers.get('set-cookies') print(cookies) def update_cookies_ZILLOW_SID(self): cookies = self.cookies headers = { 'accept': '*/*', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'pragma': 'no-cache', 'priority': 'u=1, i', 'referer': 'https://www.zillow.com/homedetails/3531-Bronxwood-Ave-APT-3F-Bronx-NY-10469/2053005716_zpid/', 'sec-ch-ua': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36', } params = { 'featureFlags': [ 'SHOPPING_ZOAM_MIGRATION_FOR_SALE_HDP', 'CIAM_ZOAM_MIGRATION_GLOBAL', ], } response = requests.get('https://www.zillow.com/api/user/featureFlags', params=params, cookies=cookies, headers=headers, proxies=self.proxies) scookies = response.headers['set-cookie'].split(';')[0].split('=')[1] print(scookies) self.cookies['ZILLOW_SID'] = scookies def get_search_page_info(self): url = "/async-create-search-page-state" cookies = self.cookies headers = { 'accept': '*/*', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'content-type': 'application/json', 'origin': 'https://www.zillow.com', 'pragma': 'no-cache', 'priority': 'u=1, i', # 'referer': 'https://www.zillow.com/new-york-ny/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22isMapVisible%22%3Atrue%2C%22mapBounds%22%3A%7B%22west%22%3A-73.99487916636572%2C%22east%22%3A-73.69962159800635%2C%22south%22%3A40.799450997125795%2C%22north%22%3A40.976203993908605%7D%2C%22mapZoom%22%3A12%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A6181%2C%22regionType%22%3A6%7D%5D%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D', 'sec-ch-ua': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36', } json_data = { 'searchQueryState': { 'pagination': {}, 'isMapVisible': True, 'mapBounds': { 'west': -73.99487916636572, 'east': -73.69962159800635, 'south': 40.79945099712579, 'north': 40.97620399390861, }, 'mapZoom': 12, 'regionSelection': [ { 'regionId': 6181, 'regionType': 6, }, ], 'filterState': { 'sortSelection': { 'value': 'globalrelevanceex', }, }, 'isListVisible': True, }, 'wants': { 'cat1': [ 'mapResults', ], }, 'requestId': 2, 'isDebugRequest': False, } response = requests.put(url=self.baseurl + url, cookies=cookies, headers=headers, json=json_data, proxies=self.proxies) json_data = response.json() return json_data def json_parsing(self, search_json): for i in search_json.get("cat1").get("searchResults").get("mapResults"): detailurl = i.get("detailUrl") if "homedetails" in detailurl: self.get_homedetails(i.get("detailUrl")) def get_homedetails(self, detailUrl, r=3): cookies = self.cookies headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'accept-language': 'zh-CN,zh;q=0.9', 'cache-control': 'no-cache', 'pragma': 'no-cache', 'priority': 'u=0, i', 'sec-ch-ua': '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', 'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', 'sec-fetch-user': '?1', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36' } if r > 0: try: response = requests.get(url=self.baseurl + detailUrl, cookies=cookies, headers=headers, proxies=self.proxies) data_json_str = re.findall(r'\"gdpClientCache\":(.*?),\"composedGraphQLQuery\"', response.text)[0] except: time.sleep(2) print(response.status_code) print(self.baseurl + detailUrl) self.get_JSESSIONID() if r == 3: with open('error_list.txt', 'w', encoding='utf-8') as f: f.write(self.baseurl + detailUrl) return self.get_homedetails(detailUrl, r - 1) else: print("重试最大次数!") return if response.headers.get('set-cookie'): scookies = response.headers['set-cookie'].split(';')[0].split('=')[1] self.cookies['ZILLOW_SID'] = scookies data_json_str = data_json_str.encode('utf-8').decode('unicode_escape') # 去除转义 data_json = ('{"data":{"property":' + data_json_str.split(':{"property":')[-1])[:-1] jd = json.loads(data_json).get("data").get("property") new_row = [ jd.get("zpid"), jd.get("price"), jd.get("yearBuilt"), jd.get("address").get("streetAddress"), jd.get("bedrooms"), jd.get("bathrooms"), "|".join([i.get('mixedSources').get('jpeg')[-1].get('url') for i in jd.get("responsivePhotos")]), jd.get('timeOnZillow'), jd.get('pageViewCount'), jd.get('favoriteCount'), '' if not jd.get("homeInsights") else "|".join( [i for i in jd.get("homeInsights")[0].get("insights")[0].get("phrases")]), jd.get("description") ] with open('data.csv', 'a', newline='', encoding='utf-8') as f: writer = csv.writer(f) writer.writerow(new_row) print(f"成功保存：{new_row[0]}") if __name__ == '__main__': Z = Zillow() search_data = Z.get_search_page_info() Z.json_parsing(search_data) # Z.get_JSESSIONID()