SMA Sunfire to csv
View all Actors
This Actor is deprecated
This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?
See alternative ActorsSMA Sunfire to csv
medicare_tools/sma-sunfire-to-csv
Extract your enrollments in SMA's Sunfire (aka Medicare Plan Finder) into a csv file.
.actor/Dockerfile
1# First, specify the base Docker image.
2# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
3# You can also use any other image from Docker Hub.
4FROM apify/actor-python-selenium:3.11
5
6# Second, copy just requirements.txt into the Actor image,
7# since it should be the only file that affects the dependency install in the next step,
8# in order to speed up the build
9COPY requirements.txt ./
10
11# Install the packages specified in requirements.txt,
12# Print the installed Python version, pip version
13# and all installed packages with their versions for debugging
14RUN echo "Python version:" \
15 && python --version \
16 && echo "Pip version:" \
17 && pip --version \
18 && echo "Installing dependencies:" \
19 && pip install -r requirements.txt \
20 && echo "All installed Python packages:" \
21 && pip freeze
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after installing the dependencies, quick build will be really fast
25# for most source file changes.
26COPY . ./
27
28# Specify how to launch the source code of your Actor.
29# By default, the "python3 -m src" command is run
30CMD ["python3", "-m", "src"].actor/actor.json
1{
2 "actorSpecification": 1,
3 "name": "SunfireMatrix-actor",
4 "title": "Data Extraction from SunfireMatrix",
5 "description": "Extracts client data from SunfireMatrix using credentials.",
6 "version": "0.1",
7 "meta": {
8 "templateId": "python-selenium"
9 },
10 "input": "./input_schema.json",
11 "dockerfile": "./Dockerfile",
12 "storages": {
13 "dataset": {
14 "actorSpecification": 1,
15 "title": "SunfireMatrix Client Data",
16 "views": {
17 "clientData": {
18 "title": "SunfireMatrix Client Data",
19 "transformation": {
20 "fields": [
21 "firstName",
22 "lastName",
23 "PCP",
24 "monthlyPremium",
25 "effectiveDate",
26 "DOB",
27 "gender",
28 "address",
29 "ethnicity",
30 "race",
31 "language",
32 "MBI",
33 "electionPeriod",
34 "Part_A_effective_date",
35 "Part_B_effective_date",
36 "sepCode",
37 "enrollmentCode",
38 "agentName",
39 "writingId"
40 ]
41 },
42 "display": {
43 "component": "table",
44 "properties": {
45 "firstName": {
46 "label": "First Name",
47 "format": "text"
48 },
49 "lastName": {
50 "label": "Last Name",
51 "format": "text"
52 },
53 "PCP": {
54 "label": "Primary Care Physician/Plan Name",
55 "format": "text"
56 },
57 "monthlyPremium": {
58 "label": "Monthly Premium",
59 "format": "text"
60 },
61 "effectiveDate": {
62 "label": "Effective Date",
63 "format": "text"
64 },
65 "DOB": {
66 "label": "Date of Birth",
67 "format": "text"
68 },
69 "gender": {
70 "label": "Gender",
71 "format": "text"
72 },
73 "address": {
74 "label": "Address",
75 "format": "text"
76 },
77 "ethnicity": {
78 "label": "Ethnicity",
79 "format": "text"
80 },
81 "race": {
82 "label": "Race",
83 "format": "text"
84 },
85 "language": {
86 "label": "Preferred Language",
87 "format": "text"
88 },
89 "MBI": {
90 "label": "Medicare Beneficiary Identifier",
91 "format": "text"
92 },
93 "electionPeriod": {
94 "label": "Election Period",
95 "format": "text"
96 },
97 "Part_A_effective_date": {
98 "label": "Part A Effective Date",
99 "format": "text"
100 },
101 "Part_B_effective_date": {
102 "label": "Part B Effective Date",
103 "format": "text"
104 },
105 "sepCode": {
106 "label": "SEP Code",
107 "format": "text"
108 },
109 "enrollmentCode": {
110 "label": "Enrollment Code",
111 "format": "text"
112 },
113 "agentName": {
114 "label": "Agent Name",
115 "format": "text"
116 },
117 "writingId": {
118 "label": "Writing ID",
119 "format": "text"
120 }
121 }
122 }
123 }
124 }
125 }
126 }
127}.actor/input_schema.json
1{
2 "title": "SunfireMatrix",
3 "type": "object",
4 "schemaVersion": 1,
5 "properties": {
6 "username": {
7 "title": "Username/Email",
8 "type": "string",
9 "description": "Enter the username or email address used for login.",
10 "editor": "textfield"
11 },
12 "password": {
13 "title": "Password",
14 "type": "string",
15 "description": "Enter the password for login.",
16 "isSecret": true,
17 "editor": "textfield"
18 },
19 "start_date":{
20 "title":"Enrollments from",
21 "type":"string",
22 "description":"Enter the Starting Date",
23 "editor": "datepicker"
24 },
25 "end_date":{
26 "title":"To",
27 "type":"string",
28 "description":"Enter the Ending Date",
29 "editor":"datepicker"
30 }
31 },
32 "required": ["username", "password"]
33}src/__main__.py
1import asyncio
2import logging
3
4from apify.log import ActorLogFormatter
5
6from .main import main
7
8handler = logging.StreamHandler()
9handler.setFormatter(ActorLogFormatter())
10
11apify_client_logger = logging.getLogger('apify_client')
12apify_client_logger.setLevel(logging.INFO)
13apify_client_logger.addHandler(handler)
14
15apify_logger = logging.getLogger('apify')
16apify_logger.setLevel(logging.DEBUG)
17apify_logger.addHandler(handler)
18
19asyncio.run(main())src/main.py
1from urllib.parse import urljoin
2from selenium import webdriver
3from selenium.webdriver.chrome.options import Options as ChromeOptions
4from selenium.webdriver.common.by import By
5from apify import Actor
6import time
7import random
8from selenium.webdriver.support.ui import Select
9from selenium.webdriver.common.keys import Keys
10from selenium.webdriver.common.by import By
11import requests
12from bs4 import BeautifulSoup as bs
13import csv
14import pandas as pd
15from tqdm import tqdm
16from datetime import datetime, timedelta
17import dateutil.relativedelta
18
19
20async def login(browser, username, password):
21 browser.get("https://www.sunfirematrix.com/app/agent/sma/")
22 time.sleep(random.uniform(4, 5))
23 Actor.log.info(browser.title)
24 # send npi
25 browser.find_element(By.CSS_SELECTOR, 'input[name="username"]').send_keys(username)
26 # send password
27 browser.find_element(By.CSS_SELECTOR, 'input[name="password"]').send_keys(password)
28 # click login
29 browser.find_element(By.CSS_SELECTOR, 'button[type="submit"]').click()
30 time.sleep(random.uniform(8, 10))
31 # click reporting
32
33async def open_report(browser):
34 try:
35 browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
36 except:
37 try:
38 browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
39 except:
40 pass
41 time.sleep(random.uniform(12, 15))
42 # html = browser.page_source
43 # save to html file
44 timeframe = Select(browser.find_element(By.CSS_SELECTOR, 'select[id="timeFrame"]'))
45 timeframe.select_by_value("custom")
46 today = datetime.today().strftime("%m/%d/%Y")
47 # make two months ago
48 two_months_ago = datetime.today() - pd.DateOffset(months=2)
49 two_months_ago = two_months_ago.strftime("%m/%d/%Y")
50 startDate = browser.find_element(By.CSS_SELECTOR, 'input[name="startDate"]')
51 startDate.clear()
52 startDate.send_keys(two_months_ago)
53 startDate.send_keys(Keys.ENTER)
54 # send end date. mm/dd/yyyy
55 endDate = browser.find_element(By.CSS_SELECTOR, 'input[name="endDate"]')
56 endDate.clear()
57 endDate.send_keys(today)
58 endDate.send_keys(Keys.ENTER)
59 browser.find_element(By.XPATH, '//button[text()="Run"]').click()
60 time.sleep(random.uniform(4, 6))
61
62async def get_auth(cookies):
63 headers = {
64 'authority': 'www.sunfirematrix.com',
65 'accept': '*/*',
66 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
67 'authorization': cookies['rh'],
68 'content-type': 'application/json',
69 'referer': 'https://www.sunfirematrix.com/app/agent/int/',
70 'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
71 'sec-ch-ua-mobile': '?0',
72 'sec-ch-ua-platform': '"Windows"',
73 'sec-fetch-dest': 'empty',
74 'sec-fetch-mode': 'cors',
75 'sec-fetch-site': 'same-origin',
76 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
77 }
78
79 response = requests.get('https://www.sunfirematrix.com/api/partner/info', cookies=cookies, headers=headers)
80 info = response.json()
81 auth = info['clientPartnerId']
82 npn = info['npn']
83 return auth, npn
84
85
86async def codes(cookies, auth, username, start_date, end_date):
87 headers = {
88 'authority': 'www.sunfirematrix.com',
89 'accept': '*/*',
90 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
91 'authorization': cookies['rh'],
92 'content-type': 'application/json',
93 'referer': 'https://www.sunfirematrix.com/app/agent/int/',
94 'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
95 'sec-ch-ua-mobile': '?0',
96 'sec-ch-ua-platform': '"Windows"',
97 'sec-fetch-dest': 'empty',
98 'sec-fetch-mode': 'cors',
99 'sec-fetch-site': 'same-origin',
100 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
101 }
102 today = datetime.today().strftime("%Y-%m-%d")
103 # make two months ago
104 two_months_ago = datetime.today() - pd.DateOffset(months=2)
105 two_months_ago = two_months_ago.strftime("%Y-%m-%d")
106
107 # make two months batch from start_date and end_date
108 # loop through the batch and get codes
109
110 start_date = datetime.strptime(start_date, '%Y-%m-%d')
111 end_date = datetime.strptime(end_date, '%Y-%m-%d')
112 codes = []
113 while start_date < end_date:
114 two_months_later = start_date + dateutil.relativedelta.relativedelta(months=2)
115 response = requests.get(
116 f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{username}@sunfire.integritymarketinggroup.com/{start_date}T18%3A00%3A00.000Z/{two_months_later}T18%3A00%3A00.000Z',
117 cookies=cookies,
118 headers=headers,
119 )
120 enrollmentCode = response.json()
121 enrollmentCode = [a['enrollmentCode'] for a in enrollmentCode]
122 codes.extend(enrollmentCode)
123 Actor.log.info(f"Got {len(enrollmentCode)} codes from {start_date.isoformat()} to {two_months_later.isoformat()}")
124 # process the response here
125 start_date = two_months_later
126 # response = requests.get(
127 # f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{npi_number}@sunfire.integritymarketinggroup.com/{two_months_ago}T18%3A00%3A00.000Z/{today}T18%3A00%3A00.000Z',
128 # cookies=cookies,
129 # headers=headers,
130 # )
131
132 return codes
133
134
135async def get_codes(browser, username, start_date, end_date):
136 cookies = browser.get_cookies()
137 cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
138 auth, username = await get_auth(cookies)
139 enrollmentCode = await codes(cookies, auth, username, start_date, end_date)
140 return enrollmentCode
141
142
143async def process(data):
144 client_data = data
145 form_data = client_data['formData']
146 firstName = form_data['firstName']
147 lastName = form_data['lastName']
148 try:
149 planName = form_data['plan']['displayName']
150 except:
151 planName = ''
152 try:
153 monthlyPremium = form_data['plan']['fmtPremium']
154 except:
155 monthlyPremium = ''
156 try:
157 effectiveDate = form_data['effectiveDate']
158 effectiveDate = effectiveDate['month'] + '-' + effectiveDate['day'] + '-' + effectiveDate['year']
159 except:
160 effectiveDate = ''
161 dob = form_data['dob']
162 dob = dob['month'] + '-' + dob['day'] + '-' + dob['year']
163 gender = form_data['gender']
164 # address = form_data['home']
165 address_fields = ['line1', 'line2', 'city', 'countyName', 'state', 'zip']
166 address = form_data.get('home', {})
167 address_str = ', '.join([address.get(field, '') for field in address_fields])
168 # check i
169 # address = address['line1'] + ', ' + address['line2'] + ', ' + address['city'] + ', ' + address['countyName'] +', '+ address['state'] + ', ' + address['zip']
170 raceEthnicitySpecify = form_data['raceEthnicitySpecify']
171 try:
172 ethnicity = raceEthnicitySpecify['ethnicity']
173 except:
174 ethnicity = ''
175 try:
176 race = raceEthnicitySpecify['race']
177 except:
178 race = ''
179 try:
180 language = form_data['language']
181 except:
182 language = ''
183 medicareNumber = form_data['medicareNumber']
184 try:
185 Part_A_effective_date = form_data['partADate']
186 Part_A_effective_date = Part_A_effective_date['month'] + '-' + Part_A_effective_date['day'] + '-' + Part_A_effective_date['year']
187 except:
188 Part_A_effective_date = ''
189
190 try:
191 Part_B_effective_date = form_data['partBDate']
192 Part_B_effective_date = Part_B_effective_date['month'] + '-' + Part_B_effective_date['day'] + '-' + Part_B_effective_date['year']
193 except:
194 Part_B_effective_date = ''
195
196 electionPeriod = form_data['electionPeriod']
197 try:
198 sepCode = form_data['sepCode']
199 except:
200 sepCode = 'N/A'
201 try:
202 planName = form_data['pcp']['name']
203 except:
204 planName = 'N/A'
205 enrollmentCode = form_data['enrollmentCode']
206 agentName = form_data['agentModel']['firstName'] + ' ' + form_data['agentModel']['lastName']
207 writingId = form_data['agentModel']['writingId']
208
209
210 client_info = {
211 "firstName": firstName,
212 "lastName": lastName,
213 "PCP": planName,
214 "monthlyPremium": monthlyPremium,
215 "effectiveDate": effectiveDate,
216 "DOB": dob,
217 "gender": gender,
218 "address": address_str,
219 "ethnicity": ethnicity,
220 "race": race,
221 "language": language,
222 "MBI": medicareNumber,
223 "electionPeriod": electionPeriod,
224 "Part_A_effective_date": Part_A_effective_date,
225 "Part_B_effective_date": Part_B_effective_date,
226 "sepCode": sepCode,
227 "enrollmentCode": enrollmentCode,
228 "agentName": agentName,
229 "writingId": writingId
230 }
231 # Actor.push_data(client_info)
232 # with open(f'{npi_numer}_client_info.json', 'a', encoding='utf-8') as f:
233 # json.dump(client_info, f, ensure_ascii=False, indent=4)
234 # # add comma
235 # f.write(',\n')
236 return client_info
237 # print(client_info)
238
239async def get_report(enrollmentCode, cookies, username):
240 headers = {
241 'authority': 'www.sunfirematrix.com',
242 'accept': '*/*',
243 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
244 'authorization': cookies['rh'],
245 'content-type': 'application/json',
246 'referer': 'https://www.sunfirematrix.com/app/agent/int/?scope=openid%20profile%20email&session_state=c8-66PcTpUvG7cD247EYXmocEQ2M_Sm5nXKeoc5CQLs.86DD30C6A324B3E309C9DFF6E83101D7',
247 'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
248 'sec-ch-ua-mobile': '?0',
249 'sec-ch-ua-platform': '"Windows"',
250 'sec-fetch-dest': 'empty',
251 'sec-fetch-mode': 'cors',
252 'sec-fetch-site': 'same-origin',
253 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
254 }
255
256 response = requests.get(
257 f'https://www.sunfirematrix.com/api/form/mc/load/pdf/{username}%40sunfire.integritymarketinggroup.com/{enrollmentCode}',
258 cookies=cookies,
259 headers=headers,
260 )
261 client_info = await process(response.json())
262 return client_info
263
264
265
266async def main():
267 async with Actor:
268 # Read the Actor input
269 actor_input = await Actor.get_input() or {}
270 username = actor_input.get("username")
271 password = actor_input.get("password")
272 start_date = actor_input.get("start_date")
273 end_date = actor_input.get("end_date")
274 # check both start_date and end_date are not empty
275 # if they are empty, set start_date to 2 months ago and end_date to today
276 if not start_date and not end_date:
277 # make two months ago
278 two_months_ago = datetime.today() - pd.DateOffset(months=2)
279 two_months_ago = two_months_ago.strftime("%Y-%m-%d")
280 start_date = two_months_ago
281 end_date = datetime.today().strftime("%Y-%m-%d")
282
283 # Launch a new Selenium Chrome WebDriver
284 Actor.log.info('Launching Chrome WebDriver...')
285 chrome_options = ChromeOptions()
286 if Actor.config.headless:
287 chrome_options.add_argument('--headless')
288 chrome_options.add_argument('--no-sandbox')
289 chrome_options.add_argument('--disable-dev-shm-usage')
290 chrome_options.add_argument('--window-size=1920,1080')
291 browser = webdriver.Chrome(options=chrome_options)
292 # ruh the login function
293 await login(browser, username, password)
294 Actor.log.info("Logged In!")
295 # open report
296 await open_report(browser)
297 Actor.log.info("Report Opened!")
298 # get auth
299 enrollmentCode = await get_codes(browser, username, start_date, end_date)
300 Actor.log.info("Got Enrollment Codes!")
301 cookies = browser.get_cookies()
302 cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
303 # client_info = []
304 for code in tqdm(enrollmentCode):
305 Actor.log.info("Getting Data....")
306 client_info = await get_report(code, cookies, username)
307 await Actor.push_data(client_info)
308 # client_info.append(await get_report(code, cookies, npi_number))
309 Actor.log.info("Completed.")
310 browser.quit().dockerignore
1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10.venv
11
12# git folder
13.git.editorconfig
1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf.gitignore
1# This file tells Git which files shouldn't be added to source control
2
3.idea
4.DS_Store
5
6apify_storage
7storage
8
9.venv/
10.env/
11__pypackages__
12dist/
13build/
14*.egg-info/
15*.egg
16
17__pycache__
18
19.mypy_cache
20.dmypy.json
21dmypy.json
22.pytest_cache
23
24.scrapy
25*.logrequirements.txt
1# Add your dependencies here.
2# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
3# for how to format them
4apify ~= 1.2.0
5selenium ~= 4.14.0
6seleniumbase
7tqdm
8bs4
9pandasDeveloper
Medicare ToolsMaintained by Community
Categories
