Deprecated

This Actor is deprecated

This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?

See alternative Actors

IFG Sunfire to csv

medicare_tools/ifg-sunfire-to-csv

Extract your enrollments in IFG's Sunfire into a csv file.

.actor/Dockerfile

1# First, specify the base Docker image.
2# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
3# You can also use any other image from Docker Hub.
4FROM apify/actor-python-selenium:3.11
5
6# Second, copy just requirements.txt into the Actor image,
7# since it should be the only file that affects the dependency install in the next step,
8# in order to speed up the build
9COPY requirements.txt ./
10
11# Install the packages specified in requirements.txt,
12# Print the installed Python version, pip version
13# and all installed packages with their versions for debugging
14RUN echo "Python version:" \
15 && python --version \
16 && echo "Pip version:" \
17 && pip --version \
18 && echo "Installing dependencies:" \
19 && pip install -r requirements.txt \
20 && echo "All installed Python packages:" \
21 && pip freeze
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after installing the dependencies, quick build will be really fast
25# for most source file changes.
26COPY . ./
27
28# Specify how to launch the source code of your Actor.
29# By default, the "python3 -m src" command is run
30CMD ["python3", "-m", "src"]

.actor/actor.json

1{
2    "actorSpecification": 1,
3    "name": "SunfireMatrix-actor",
4    "title": "Data Extraction from SunfireMatrix",
5    "description": "Extracts client data from SunfireMatrix using credentials.",
6    "version": "0.1",
7    "meta": {
8        "templateId": "python-selenium"
9    },
10    "input": "./input_schema.json",
11    "dockerfile": "./Dockerfile",
12    "storages": {
13        "dataset": {
14            "actorSpecification": 1,
15            "title": "SunfireMatrix Client Data",
16            "views": {
17                "clientData": {
18                    "title": "SunfireMatrix Client Data",
19                    "transformation": {
20                        "fields": [
21                            "firstName",
22                            "lastName",
23                            "PCP",
24                            "monthlyPremium",
25                            "effectiveDate",
26                            "DOB",
27                            "gender",
28                            "address",
29                            "ethnicity",
30                            "race",
31                            "language",
32                            "MBI",
33                            "electionPeriod",
34                            "Part_A_effective_date",
35                            "Part_B_effective_date",
36                            "sepCode",
37                            "enrollmentCode",
38                            "agentName",
39                            "writingId"
40                        ]
41                    },
42                    "display": {
43                        "component": "table",
44                        "properties": {
45                            "firstName": {
46                                "label": "First Name",
47                                "format": "text"
48                            },
49                            "lastName": {
50                                "label": "Last Name",
51                                "format": "text"
52                            },
53                            "PCP": {
54                                "label": "Primary Care Physician/Plan Name",
55                                "format": "text"
56                            },
57                            "monthlyPremium": {
58                                "label": "Monthly Premium",
59                                "format": "text"
60                            },
61                            "effectiveDate": {
62                                "label": "Effective Date",
63                                "format": "text"
64                            },
65                            "DOB": {
66                                "label": "Date of Birth",
67                                "format": "text"
68                            },
69                            "gender": {
70                                "label": "Gender",
71                                "format": "text"
72                            },
73                            "address": {
74                                "label": "Address",
75                                "format": "text"
76                            },
77                            "ethnicity": {
78                                "label": "Ethnicity",
79                                "format": "text"
80                            },
81                            "race": {
82                                "label": "Race",
83                                "format": "text"
84                            },
85                            "language": {
86                                "label": "Preferred Language",
87                                "format": "text"
88                            },
89                            "MBI": {
90                                "label": "Medicare Beneficiary Identifier",
91                                "format": "text"
92                            },
93                            "electionPeriod": {
94                                "label": "Election Period",
95                                "format": "text"
96                            },
97                            "Part_A_effective_date": {
98                                "label": "Part A Effective Date",
99                                "format": "text"
100                            },
101                            "Part_B_effective_date": {
102                                "label": "Part B Effective Date",
103                                "format": "text"
104                            },
105                            "sepCode": {
106                                "label": "SEP Code",
107                                "format": "text"
108                            },
109                            "enrollmentCode": {
110                                "label": "Enrollment Code",
111                                "format": "text"
112                            },
113                            "agentName": {
114                                "label": "Agent Name",
115                                "format": "text"
116                            },
117                            "writingId": {
118                                "label": "Writing ID",
119                                "format": "text"
120                            }
121                        }
122                    }
123                }
124            }
125        }
126    }
127}

.actor/input_schema.json

1{
2  "title": "SunfireMatrix",
3  "type": "object",
4  "schemaVersion": 1,
5  "properties": {
6    "username": {
7      "title": "Username/Email",
8      "type": "string",
9      "description": "Enter the username or email address used for login.",
10      "editor": "textfield"
11    },
12    "password": {
13      "title": "Password",
14      "type": "string",
15      "description": "Enter the password for login.",
16      "isSecret": true,
17      "editor": "textfield"
18    },
19    "start_date":{
20      "title":"Enrollments from",
21      "type":"string",
22      "description":"Enter the Starting Date",
23      "editor": "datepicker"
24    },
25    "end_date":{
26      "title":"To",
27      "type":"string",
28      "description":"Enter the Ending Date",
29      "editor":"datepicker"
30    }
31  },
32  "required": ["username", "password"]
33}

src/main.py

1import asyncio
2import logging
3
4from apify.log import ActorLogFormatter
5
6from .main import main
7
8handler = logging.StreamHandler()
9handler.setFormatter(ActorLogFormatter())
10
11apify_client_logger = logging.getLogger('apify_client')
12apify_client_logger.setLevel(logging.INFO)
13apify_client_logger.addHandler(handler)
14
15apify_logger = logging.getLogger('apify')
16apify_logger.setLevel(logging.DEBUG)
17apify_logger.addHandler(handler)
18
19asyncio.run(main())

src/main.py

1from urllib.parse import urljoin
2from selenium import webdriver
3from selenium.webdriver.chrome.options import Options as ChromeOptions
4from selenium.webdriver.common.by import By
5from apify import Actor
6import time
7import random
8from selenium.webdriver.support.ui import Select
9from selenium.webdriver.common.keys import Keys
10from selenium.webdriver.common.by import By
11import requests
12from bs4 import BeautifulSoup as bs
13import csv
14import pandas as pd
15from tqdm import tqdm
16from datetime import datetime, timedelta
17import dateutil.relativedelta
18
19
20async def login(browser, username, password):
21    browser.get("https://www.sunfirematrix.com/app/agent/ifg")
22    time.sleep(random.uniform(4, 5))
23    Actor.log.info(browser.title)
24    # send npi
25    browser.find_element(By.CSS_SELECTOR, 'input[name="username"]').send_keys(username)
26    # send password
27    browser.find_element(By.CSS_SELECTOR, 'input[name="password"]').send_keys(password)
28    # click login
29    browser.find_element(By.CSS_SELECTOR, 'button[type="submit"]').click()
30    time.sleep(random.uniform(8, 10))
31    # click reporting
32
33async def open_report(browser):
34    try:
35        browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
36    except:
37        try:
38            browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
39        except:
40            pass
41    time.sleep(random.uniform(12, 15))
42    # html = browser.page_source
43    # save to html file
44    timeframe = Select(browser.find_element(By.CSS_SELECTOR, 'select[id="timeFrame"]'))
45    timeframe.select_by_value("custom")
46    today  = datetime.today().strftime("%m/%d/%Y")
47    # make two months ago
48    two_months_ago = datetime.today() - pd.DateOffset(months=2)
49    two_months_ago = two_months_ago.strftime("%m/%d/%Y")
50    startDate = browser.find_element(By.CSS_SELECTOR, 'input[name="startDate"]')
51    startDate.clear()
52    startDate.send_keys(two_months_ago)
53    startDate.send_keys(Keys.ENTER)
54    # send end date. mm/dd/yyyy
55    endDate = browser.find_element(By.CSS_SELECTOR, 'input[name="endDate"]')
56    endDate.clear()
57    endDate.send_keys(today)
58    endDate.send_keys(Keys.ENTER)
59    browser.find_element(By.XPATH, '//button[text()="Run"]').click()
60    time.sleep(random.uniform(4, 6))
61
62async def get_auth(cookies):
63    headers = {
64        'authority': 'www.sunfirematrix.com',
65        'accept': '*/*',
66        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
67        'authorization': cookies['rh'],
68        'content-type': 'application/json',
69        'referer': 'https://www.sunfirematrix.com/app/agent/int/',
70        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
71        'sec-ch-ua-mobile': '?0',
72        'sec-ch-ua-platform': '"Windows"',
73        'sec-fetch-dest': 'empty',
74        'sec-fetch-mode': 'cors',
75        'sec-fetch-site': 'same-origin',
76        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
77    }
78
79    response = requests.get('https://www.sunfirematrix.com/api/partner/info', cookies=cookies, headers=headers)
80    info = response.json()
81    auth = info['clientPartnerId']
82    npn = info['npn']
83    return auth, npn
84
85
86async def codes(cookies, auth, username, start_date, end_date):
87    headers = {
88        'authority': 'www.sunfirematrix.com',
89        'accept': '*/*',
90        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
91        'authorization': cookies['rh'],
92        'content-type': 'application/json',
93        'referer': 'https://www.sunfirematrix.com/app/agent/int/',
94        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
95        'sec-ch-ua-mobile': '?0',
96        'sec-ch-ua-platform': '"Windows"',
97        'sec-fetch-dest': 'empty',
98        'sec-fetch-mode': 'cors',
99        'sec-fetch-site': 'same-origin',
100        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
101    }
102    today = datetime.today().strftime("%Y-%m-%d")
103    # make two months ago
104    two_months_ago = datetime.today() - pd.DateOffset(months=2)
105    two_months_ago = two_months_ago.strftime("%Y-%m-%d")
106
107    # make two months batch from start_date and end_date
108    # loop through the batch and get codes
109
110    start_date = datetime.strptime(start_date, '%Y-%m-%d')
111    end_date = datetime.strptime(end_date, '%Y-%m-%d')
112    codes = []
113    while start_date < end_date:
114        two_months_later = start_date + dateutil.relativedelta.relativedelta(months=2)
115        response = requests.get(
116            f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{username}@sunfire.integritymarketinggroup.com/{start_date}T18%3A00%3A00.000Z/{two_months_later}T18%3A00%3A00.000Z',
117            cookies=cookies,
118            headers=headers,
119        )
120        enrollmentCode = response.json()
121        enrollmentCode = [a['enrollmentCode'] for a in enrollmentCode]
122        codes.extend(enrollmentCode)
123        Actor.log.info(f"Got {len(enrollmentCode)} codes from {start_date.isoformat()} to {two_months_later.isoformat()}")
124        # process the response here
125        start_date = two_months_later
126    # response = requests.get(
127    #     f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{npi_number}@sunfire.integritymarketinggroup.com/{two_months_ago}T18%3A00%3A00.000Z/{today}T18%3A00%3A00.000Z',
128    #     cookies=cookies,
129    #     headers=headers,
130    # )
131
132    return codes
133
134
135async def get_codes(browser, username, start_date, end_date):
136    cookies = browser.get_cookies()
137    cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
138    auth, username = await get_auth(cookies)
139    enrollmentCode = await codes(cookies, auth, username, start_date, end_date)
140    return enrollmentCode
141
142
143async def process(data):
144    client_data = data
145    form_data = client_data['formData']
146    firstName = form_data['firstName']
147    lastName = form_data['lastName']
148    try:
149        planName = form_data['plan']['displayName']
150    except:
151        planName = ''
152    try:
153        monthlyPremium = form_data['plan']['fmtPremium']
154    except:
155        monthlyPremium = ''
156    try:
157        effectiveDate = form_data['effectiveDate']
158        effectiveDate = effectiveDate['month'] + '-' + effectiveDate['day'] + '-' + effectiveDate['year']
159    except:
160        effectiveDate = ''
161    dob = form_data['dob']
162    dob = dob['month'] + '-' + dob['day'] + '-' + dob['year']
163    gender = form_data['gender']
164    # address = form_data['home']
165    address_fields = ['line1', 'line2', 'city', 'countyName', 'state', 'zip']
166    address = form_data.get('home', {})
167    address_str = ', '.join([address.get(field, '') for field in address_fields])
168    # check i
169    # address = address['line1'] + ', ' + address['line2'] + ', ' + address['city'] + ', ' + address['countyName'] +', '+ address['state'] + ', ' + address['zip']
170    raceEthnicitySpecify = form_data['raceEthnicitySpecify']
171    try:
172        ethnicity = raceEthnicitySpecify['ethnicity']
173    except:
174        ethnicity = ''
175    try:
176        race = raceEthnicitySpecify['race']
177    except:
178        race = ''
179    try:
180        language = form_data['language']
181    except:
182        language = ''
183    medicareNumber = form_data['medicareNumber']
184    try:
185        Part_A_effective_date = form_data['partADate']
186        Part_A_effective_date = Part_A_effective_date['month'] + '-' + Part_A_effective_date['day'] + '-' + Part_A_effective_date['year']
187    except:
188        Part_A_effective_date = ''
189
190    try:
191        Part_B_effective_date = form_data['partBDate']
192        Part_B_effective_date = Part_B_effective_date['month'] + '-' + Part_B_effective_date['day'] + '-' + Part_B_effective_date['year']
193    except:
194        Part_B_effective_date = ''
195
196    electionPeriod = form_data['electionPeriod']
197    try:
198        sepCode = form_data['sepCode']
199    except:
200        sepCode = 'N/A'
201    try:
202        planName = form_data['pcp']['name']
203    except:
204        planName = 'N/A'
205    enrollmentCode = form_data['enrollmentCode']
206    agentName = form_data['agentModel']['firstName'] + ' ' + form_data['agentModel']['lastName']
207    writingId = form_data['agentModel']['writingId']
208
209
210    client_info = {
211        "firstName": firstName,
212        "lastName": lastName,
213        "PCP": planName,
214        "monthlyPremium": monthlyPremium,
215        "effectiveDate": effectiveDate,
216        "DOB": dob,
217        "gender": gender,
218        "address": address_str,
219        "ethnicity": ethnicity,
220        "race": race,
221        "language": language,
222        "MBI": medicareNumber,
223        "electionPeriod": electionPeriod,
224        "Part_A_effective_date": Part_A_effective_date,
225        "Part_B_effective_date": Part_B_effective_date,
226        "sepCode": sepCode,
227        "enrollmentCode": enrollmentCode,
228        "agentName": agentName,
229        "writingId": writingId
230    }
231    # Actor.push_data(client_info)
232    # with open(f'{npi_numer}_client_info.json', 'a', encoding='utf-8') as f:
233    #     json.dump(client_info, f, ensure_ascii=False, indent=4)
234    #     # add comma
235    #     f.write(',\n')
236    return client_info
237    # print(client_info)
238
239async def get_report(enrollmentCode, cookies, username):
240    headers = {
241        'authority': 'www.sunfirematrix.com',
242        'accept': '*/*',
243        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
244        'authorization': cookies['rh'],
245        'content-type': 'application/json',
246        'referer': 'https://www.sunfirematrix.com/app/agent/int/?scope=openid%20profile%20email&session_state=c8-66PcTpUvG7cD247EYXmocEQ2M_Sm5nXKeoc5CQLs.86DD30C6A324B3E309C9DFF6E83101D7',
247        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
248        'sec-ch-ua-mobile': '?0',
249        'sec-ch-ua-platform': '"Windows"',
250        'sec-fetch-dest': 'empty',
251        'sec-fetch-mode': 'cors',
252        'sec-fetch-site': 'same-origin',
253        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
254    }
255
256    response = requests.get(
257        f'https://www.sunfirematrix.com/api/form/mc/load/pdf/{username}%40sunfire.integritymarketinggroup.com/{enrollmentCode}',
258        cookies=cookies,
259        headers=headers,
260    )
261    client_info = await process(response.json())
262    return client_info
263
264
265
266async def main():
267    async with Actor:
268        # Read the Actor input
269        actor_input = await Actor.get_input() or {}
270        username = actor_input.get("username")
271        password = actor_input.get("password")
272        start_date = actor_input.get("start_date")
273        end_date = actor_input.get("end_date")
274        # check both start_date and end_date are not empty
275        # if they are empty, set start_date to 2 months ago and end_date to today
276        if not start_date and not end_date:
277            # make two months ago
278            two_months_ago = datetime.today() - pd.DateOffset(months=2)
279            two_months_ago = two_months_ago.strftime("%Y-%m-%d")
280            start_date = two_months_ago
281            end_date = datetime.today().strftime("%Y-%m-%d")
282
283        # Launch a new Selenium Chrome WebDriver
284        Actor.log.info('Launching Chrome WebDriver...')
285        chrome_options = ChromeOptions()
286        if Actor.config.headless:
287            chrome_options.add_argument('--headless')
288        chrome_options.add_argument('--no-sandbox')
289        chrome_options.add_argument('--disable-dev-shm-usage')
290        chrome_options.add_argument('--window-size=1920,1080')
291        browser = webdriver.Chrome(options=chrome_options)
292        # ruh the login function
293        await login(browser, username, password)
294        Actor.log.info("Logged In!")
295        # open report
296        await open_report(browser)
297        Actor.log.info("Report Opened!")
298        # get auth
299        enrollmentCode = await get_codes(browser, username, start_date, end_date)
300        Actor.log.info("Got Enrollment Codes!")
301        cookies = browser.get_cookies()
302        cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
303        # client_info = []
304        for code in tqdm(enrollmentCode):
305            Actor.log.info("Getting Data....")
306            client_info = await get_report(code, cookies, username)
307            await Actor.push_data(client_info)
308            # client_info.append(await get_report(code, cookies, npi_number))
309        Actor.log.info("Completed.")
310        browser.quit()

.dockerignore

1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10.venv
11
12# git folder
13.git

.editorconfig

1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf

.gitignore

1# This file tells Git which files shouldn't be added to source control
2
3.idea
4.DS_Store
5
6apify_storage
7storage
8
9.venv/
10.env/
11__pypackages__
12dist/
13build/
14*.egg-info/
15*.egg
16
17__pycache__
18
19.mypy_cache
20.dmypy.json
21dmypy.json
22.pytest_cache
23
24.scrapy
25*.log

requirements.txt

1# Add your dependencies here.
2# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
3# for how to format them
4apify ~= 1.2.0
5selenium ~= 4.14.0
6seleniumbase
7tqdm
8bs4
9pandas