Deprecated

This Actor is deprecated

This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?

See alternative Actors

MedicareLINK (Sunfire) to csv

medicare_tools/medicarecenter-data-extractor

Extract your enrollments in MedicareLINK (Sunfire) into a csv file.

.actor/Dockerfile

1# First, specify the base Docker image.
2# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
3# You can also use any other image from Docker Hub.
4FROM apify/actor-python-selenium:3.11
5
6# Second, copy just requirements.txt into the Actor image,
7# since it should be the only file that affects the dependency install in the next step,
8# in order to speed up the build
9COPY requirements.txt ./
10
11# Install the packages specified in requirements.txt,
12# Print the installed Python version, pip version
13# and all installed packages with their versions for debugging
14RUN echo "Python version:" \
15 && python --version \
16 && echo "Pip version:" \
17 && pip --version \
18 && echo "Installing dependencies:" \
19 && pip install -r requirements.txt \
20 && echo "All installed Python packages:" \
21 && pip freeze
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after installing the dependencies, quick build will be really fast
25# for most source file changes.
26COPY . ./
27
28# Specify how to launch the source code of your Actor.
29# By default, the "python3 -m src" command is run
30CMD ["python3", "-m", "src"]

.actor/actor.json

1{
2    "actorSpecification": 1,
3    "name": "medicare-center-actor",
4    "title": "Medicare Center Data Extraction",
5    "description": "Extracts client data from Medicare Center using credentials.",
6    "version": "0.1",
7    "meta": {
8        "templateId": "python-selenium"
9    },
10    "input": "./input_schema.json",
11    "dockerfile": "./Dockerfile",
12    "storages": {
13        "dataset": {
14            "actorSpecification": 1,
15            "title": "Medicare Client Data",
16            "views": {
17                "clientData": {
18                    "title": "Medicare Client Data",
19                    "transformation": {
20                        "fields": [
21                            "firstName",
22                            "lastName",
23                            "PCP",
24                            "monthlyPremium",
25                            "effectiveDate",
26                            "DOB",
27                            "gender",
28                            "address",
29                            "ethnicity",
30                            "race",
31                            "language",
32                            "MBI",
33                            "electionPeriod",
34                            "Part_A_effective_date",
35                            "Part_B_effective_date",
36                            "sepCode",
37                            "enrollmentCode",
38                            "agentName",
39                            "writingId"
40                        ]
41                    },
42                    "display": {
43                        "component": "table",
44                        "properties": {
45                            "firstName": {
46                                "label": "First Name",
47                                "format": "text"
48                            },
49                            "lastName": {
50                                "label": "Last Name",
51                                "format": "text"
52                            },
53                            "PCP": {
54                                "label": "Primary Care Physician/Plan Name",
55                                "format": "text"
56                            },
57                            "monthlyPremium": {
58                                "label": "Monthly Premium",
59                                "format": "text"
60                            },
61                            "effectiveDate": {
62                                "label": "Effective Date",
63                                "format": "text"
64                            },
65                            "DOB": {
66                                "label": "Date of Birth",
67                                "format": "text"
68                            },
69                            "gender": {
70                                "label": "Gender",
71                                "format": "text"
72                            },
73                            "address": {
74                                "label": "Address",
75                                "format": "text"
76                            },
77                            "ethnicity": {
78                                "label": "Ethnicity",
79                                "format": "text"
80                            },
81                            "race": {
82                                "label": "Race",
83                                "format": "text"
84                            },
85                            "language": {
86                                "label": "Preferred Language",
87                                "format": "text"
88                            },
89                            "MBI": {
90                                "label": "Medicare Beneficiary Identifier",
91                                "format": "text"
92                            },
93                            "electionPeriod": {
94                                "label": "Election Period",
95                                "format": "text"
96                            },
97                            "Part_A_effective_date": {
98                                "label": "Part A Effective Date",
99                                "format": "text"
100                            },
101                            "Part_B_effective_date": {
102                                "label": "Part B Effective Date",
103                                "format": "text"
104                            },
105                            "sepCode": {
106                                "label": "SEP Code",
107                                "format": "text"
108                            },
109                            "enrollmentCode": {
110                                "label": "Enrollment Code",
111                                "format": "text"
112                            },
113                            "agentName": {
114                                "label": "Agent Name",
115                                "format": "text"
116                            },
117                            "writingId": {
118                                "label": "Writing ID",
119                                "format": "text"
120                            }
121                        }
122                    }
123                }
124            }
125        }
126    }
127}

.actor/input_schema.json

1{
2  "title": "Medicare Center Actor Input",
3  "type": "object",
4  "schemaVersion": 1,
5  "properties": {
6    "npi_number": {
7      "title": "NPN Number",
8      "type": "string",
9      "description": "Enter the NPN number used for login.",
10      "editor": "textfield"
11    },
12    "password": {
13      "title": "Password",
14      "type": "string",
15      "description": "Enter the password for login.",
16      "isSecret": true,
17      "editor": "textfield"
18    },
19    "start_date":{
20      "title":"Enrollments from",
21      "type":"string",
22      "description":"Enter the Starting Date",
23      "editor": "datepicker"
24    },
25    "end_date":{
26      "title":"To",
27      "type":"string",
28      "description":"Enter the Ending Date",
29      "editor":"datepicker"
30    }
31  },
32  "required": ["npi_number", "password"]
33}

src/init.py

src/main.py

1import asyncio
2import logging
3
4from apify.log import ActorLogFormatter
5
6from .main import main
7
8handler = logging.StreamHandler()
9handler.setFormatter(ActorLogFormatter())
10
11apify_client_logger = logging.getLogger('apify_client')
12apify_client_logger.setLevel(logging.INFO)
13apify_client_logger.addHandler(handler)
14
15apify_logger = logging.getLogger('apify')
16apify_logger.setLevel(logging.DEBUG)
17apify_logger.addHandler(handler)
18
19asyncio.run(main())

src/main.py

1from urllib.parse import urljoin
2from selenium import webdriver
3from selenium.webdriver.chrome.options import Options as ChromeOptions
4from selenium.webdriver.common.by import By
5from apify import Actor
6import time
7import random
8from selenium.webdriver.support.ui import Select
9from selenium.webdriver.common.keys import Keys
10from selenium.webdriver.common.by import By
11import requests
12from bs4 import BeautifulSoup as bs
13import csv
14import pandas as pd
15from tqdm import tqdm
16from datetime import datetime, timedelta
17import dateutil.relativedelta
18
19# To run this Actor locally, you need to have the Selenium Chromedriver installed.
20# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/
21# When running on the Apify platform, it is already included in the Actor's Docker image.
22
23async def login(browser, npi_number, password):
24    # browser.open("https://www.medicarecenter.com/welcome")
25    browser.get("https://identity.medicarecenter.com/login?ReturnUrl=https%3A%2F%2Fae-api.medicarecenter.com%2Fae-identity-service%2Fconnect%2Fauthorize%2Fcallback%3Fresponse_type%3Dcode%26scope%3Dopenid%2520profile%2520email%26client_id%3DSunfire-Client%26redirect_uri%3Dhttps%253A%252F%252Fwww.sunfirematrix.com%252Fapi%252Fpartner%252Fsso%252Fint%26state%3D")
26    time.sleep(random.uniform(4, 5))
27    # for login
28    try:
29        browser.find_element(By.CSS_SELECTOR, 'button[name="actionButton"]').click()
30        time.sleep(random.uniform(4, 6))
31    except:
32        pass
33    # send npi
34    browser.find_element(By.CSS_SELECTOR, 'input[id="login-username"]').send_keys(npi_number)
35    # send password
36    browser.find_element(By.CSS_SELECTOR, 'input[id="login-password"]').send_keys(password)
37    # click login
38    browser.find_element(By.CSS_SELECTOR, 'button[type="submit"]').click()
39    time.sleep(random.uniform(8, 10))
40
41async def open_report(browser):
42    try:
43        browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
44    except:
45        try:
46            browser.find_element(By.CSS_SELECTOR, 'a[data-test="dashboardReporting"]').click()
47        except:
48            pass
49    time.sleep(random.uniform(7, 8))
50    timeframe = Select(browser.find_element(By.CSS_SELECTOR, 'select[id="timeFrame"]'))
51    timeframe.select_by_value("custom")
52    today  = datetime.today().strftime("%m/%d/%Y")
53    # make two months ago
54    two_months_ago = datetime.today() - pd.DateOffset(months=2)
55    two_months_ago = two_months_ago.strftime("%m/%d/%Y")
56    startDate = browser.find_element(By.CSS_SELECTOR, 'input[name="startDate"]')
57    startDate.clear()
58    startDate.send_keys(two_months_ago)
59    startDate.send_keys(Keys.ENTER)
60    # send end date. mm/dd/yyyy
61    endDate = browser.find_element(By.CSS_SELECTOR, 'input[name="endDate"]')
62    endDate.clear()
63    endDate.send_keys(today)
64    endDate.send_keys(Keys.ENTER)
65    browser.find_element(By.XPATH, '//button[text()="Run"]').click()
66    time.sleep(random.uniform(4, 6))
67
68async def get_auth(cookies):
69    headers = {
70        'authority': 'www.sunfirematrix.com',
71        'accept': '*/*',
72        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
73        'authorization': cookies['rh'],
74        'content-type': 'application/json',
75        'referer': 'https://www.sunfirematrix.com/app/agent/int/',
76        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
77        'sec-ch-ua-mobile': '?0',
78        'sec-ch-ua-platform': '"Windows"',
79        'sec-fetch-dest': 'empty',
80        'sec-fetch-mode': 'cors',
81        'sec-fetch-site': 'same-origin',
82        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
83    }
84
85    response = requests.get('https://www.sunfirematrix.com/api/partner/info', cookies=cookies, headers=headers)
86    info = response.json()
87    auth = info['clientPartnerId']
88    return auth
89
90
91async def codes(cookies, auth, npi_number, start_date, end_date):
92    headers = {
93        'authority': 'www.sunfirematrix.com',
94        'accept': '*/*',
95        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
96        'authorization': cookies['rh'],
97        'content-type': 'application/json',
98        'referer': 'https://www.sunfirematrix.com/app/agent/int/',
99        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
100        'sec-ch-ua-mobile': '?0',
101        'sec-ch-ua-platform': '"Windows"',
102        'sec-fetch-dest': 'empty',
103        'sec-fetch-mode': 'cors',
104        'sec-fetch-site': 'same-origin',
105        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
106    }
107    today = datetime.today().strftime("%Y-%m-%d")
108    # make two months ago
109    two_months_ago = datetime.today() - pd.DateOffset(months=2)
110    two_months_ago = two_months_ago.strftime("%Y-%m-%d")
111
112    # make two months batch from start_date and end_date
113    # loop through the batch and get codes
114
115    start_date = datetime.strptime(start_date, '%Y-%m-%d')
116    end_date = datetime.strptime(end_date, '%Y-%m-%d')
117    codes = []
118    while start_date < end_date:
119        two_months_later = start_date + dateutil.relativedelta.relativedelta(months=2)
120        response = requests.get(
121            f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{npi_number}@sunfire.integritymarketinggroup.com/{start_date}T18%3A00%3A00.000Z/{two_months_later}T18%3A00%3A00.000Z',
122            cookies=cookies,
123            headers=headers,
124        )
125        enrollmentCode = response.json()
126        enrollmentCode = [a['enrollmentCode'] for a in enrollmentCode]
127        codes.extend(enrollmentCode)
128        Actor.log.info(f"Got {len(enrollmentCode)} codes from {start_date.isoformat()} to {two_months_later.isoformat()}")
129        # process the response here
130        start_date = two_months_later
131    # response = requests.get(
132    #     f'https://www.sunfirematrix.com/v2/enroll/list/agent/{auth}/{npi_number}@sunfire.integritymarketinggroup.com/{two_months_ago}T18%3A00%3A00.000Z/{today}T18%3A00%3A00.000Z',
133    #     cookies=cookies,
134    #     headers=headers,
135    # )
136
137    return codes
138
139
140async def get_codes(browser, npi_number, start_date, end_date):
141    cookies = browser.get_cookies()
142    cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
143    auth = await get_auth(cookies)
144    enrollmentCode = await codes(cookies, auth, npi_number, start_date, end_date)
145    return enrollmentCode
146
147
148async def process(data):
149    client_data = data
150    form_data = client_data['formData']
151    firstName = form_data['firstName']
152    lastName = form_data['lastName']
153    try:
154        planName = form_data['plan']['displayName']
155    except:
156        planName = ''
157    try:
158        monthlyPremium = form_data['plan']['fmtPremium']
159    except:
160        monthlyPremium = ''
161    try:
162        effectiveDate = form_data['effectiveDate']
163        effectiveDate = effectiveDate['month'] + '-' + effectiveDate['day'] + '-' + effectiveDate['year']
164    except:
165        effectiveDate = ''
166    dob = form_data['dob']
167    dob = dob['month'] + '-' + dob['day'] + '-' + dob['year']
168    gender = form_data['gender']
169    # address = form_data['home']
170    address_fields = ['line1', 'line2', 'city', 'countyName', 'state', 'zip']
171    address = form_data.get('home', {})
172    address_str = ', '.join([address.get(field, '') for field in address_fields])
173    # check i
174    # address = address['line1'] + ', ' + address['line2'] + ', ' + address['city'] + ', ' + address['countyName'] +', '+ address['state'] + ', ' + address['zip']
175    raceEthnicitySpecify = form_data['raceEthnicitySpecify']
176    try:
177        ethnicity = raceEthnicitySpecify['ethnicity']
178    except:
179        ethnicity = ''
180    try:
181        race = raceEthnicitySpecify['race']
182    except:
183        race = ''
184    try:
185        language = form_data['language']
186    except:
187        language = ''
188    medicareNumber = form_data['medicareNumber']
189    try:
190        Part_A_effective_date = form_data['partADate']
191        Part_A_effective_date = Part_A_effective_date['month'] + '-' + Part_A_effective_date['day'] + '-' + Part_A_effective_date['year']
192    except:
193        Part_A_effective_date = ''
194
195    try:
196        Part_B_effective_date = form_data['partBDate']
197        Part_B_effective_date = Part_B_effective_date['month'] + '-' + Part_B_effective_date['day'] + '-' + Part_B_effective_date['year']
198    except:
199        Part_B_effective_date = ''
200
201    electionPeriod = form_data['electionPeriod']
202    try:
203        sepCode = form_data['sepCode']
204    except:
205        sepCode = 'N/A'
206    try:
207        planName = form_data['pcp']['name']
208    except:
209        planName = 'N/A'
210    enrollmentCode = form_data['enrollmentCode']
211    agentName = form_data['agentModel']['firstName'] + ' ' + form_data['agentModel']['lastName']
212    writingId = form_data['agentModel']['writingId']
213
214
215    client_info = {
216        "firstName": firstName,
217        "lastName": lastName,
218        "PCP": planName,
219        "monthlyPremium": monthlyPremium,
220        "effectiveDate": effectiveDate,
221        "DOB": dob,
222        "gender": gender,
223        "address": address_str,
224        "ethnicity": ethnicity,
225        "race": race,
226        "language": language,
227        "MBI": medicareNumber,
228        "electionPeriod": electionPeriod,
229        "Part_A_effective_date": Part_A_effective_date,
230        "Part_B_effective_date": Part_B_effective_date,
231        "sepCode": sepCode,
232        "enrollmentCode": enrollmentCode,
233        "agentName": agentName,
234        "writingId": writingId
235    }
236    # Actor.push_data(client_info)
237    # with open(f'{npi_numer}_client_info.json', 'a', encoding='utf-8') as f:
238    #     json.dump(client_info, f, ensure_ascii=False, indent=4)
239    #     # add comma
240    #     f.write(',\n')
241    return client_info
242    # print(client_info)
243
244async def get_report(enrollmentCode, cookies, npi_number):
245    headers = {
246        'authority': 'www.sunfirematrix.com',
247        'accept': '*/*',
248        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
249        'authorization': cookies['rh'],
250        'content-type': 'application/json',
251        'referer': 'https://www.sunfirematrix.com/app/agent/int/?scope=openid%20profile%20email&session_state=c8-66PcTpUvG7cD247EYXmocEQ2M_Sm5nXKeoc5CQLs.86DD30C6A324B3E309C9DFF6E83101D7',
252        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
253        'sec-ch-ua-mobile': '?0',
254        'sec-ch-ua-platform': '"Windows"',
255        'sec-fetch-dest': 'empty',
256        'sec-fetch-mode': 'cors',
257        'sec-fetch-site': 'same-origin',
258        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
259    }
260
261    response = requests.get(
262        f'https://www.sunfirematrix.com/api/form/mc/load/pdf/{npi_number}%40sunfire.integritymarketinggroup.com/{enrollmentCode}',
263        cookies=cookies,
264        headers=headers,
265    )
266    client_info = await process(response.json())
267    return client_info
268
269
270
271async def main():
272    async with Actor:
273        # Read the Actor input
274        actor_input = await Actor.get_input() or {}
275        npi_number = actor_input.get("npi_number")
276        password = actor_input.get("password")
277        start_date = actor_input.get("start_date")
278        end_date = actor_input.get("end_date")
279
280
281        # Launch a new Selenium Chrome WebDriver
282        Actor.log.info('Launching Chrome WebDriver...')
283        chrome_options = ChromeOptions()
284        if Actor.config.headless:
285            chrome_options.add_argument('--headless')
286        chrome_options.add_argument('--no-sandbox')
287        chrome_options.add_argument('--disable-dev-shm-usage')
288        chrome_options.add_argument('--window-size=1920,1080')
289        browser = webdriver.Chrome(options=chrome_options)
290        # ruh the login function
291        await login(browser, npi_number, password)
292        Actor.log.info("Logged In!")
293        # open report
294        await open_report(browser)
295        Actor.log.info("Report Opened!")
296        # get auth
297        enrollmentCode = await get_codes(browser, npi_number, start_date, end_date)
298        Actor.log.info("Got Enrollment Codes!")
299        cookies = browser.get_cookies()
300        cookies = {cookie["name"]: cookie["value"] for cookie in cookies}
301        # client_info = []
302        for code in tqdm(enrollmentCode):
303            Actor.log.info("Getting Data....")
304            client_info = await get_report(code, cookies, npi_number)
305            await Actor.push_data(client_info)
306            # client_info.append(await get_report(code, cookies, npi_number))
307        Actor.log.info("Completed.")
308        browser.quit()

.dockerignore

1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10.venv
11
12# git folder
13.git

.editorconfig

1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf

.gitignore

1# This file tells Git which files shouldn't be added to source control
2
3.idea
4.DS_Store
5
6apify_storage
7storage
8
9.venv/
10.env/
11__pypackages__
12dist/
13build/
14*.egg-info/
15*.egg
16
17__pycache__
18
19.mypy_cache
20.dmypy.json
21dmypy.json
22.pytest_cache
23
24.scrapy
25*.log

requirements.txt

1# Add your dependencies here.
2# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
3# for how to format them
4apify ~= 1.2.0
5selenium ~= 4.14.0
6seleniumbase
7tqdm
8bs4
9pandas