Python Forum
Python Forum

Thread Rating:
  • 0 Vote(s) - 0 Average
  • 1
  • 2
  • 3
  • 4
  • 5
Thread Modes
Messy data to dict?
ck3mp
Programmer named Tim
Posts: 5
Threads: 1
Joined: Mar 2018
Reputation: 0
#1
Mar-08-2018, 05:11 PM
Hello,

I have some data that is returned as part of a call to a website. I need to get the data into a CSV file ultimatly, but I cant currently get it into a dict or other usable format...


import re
import ast
import csv

output = '//<![CDATA[window.gon={};gon.current_user={"email":"[email protected]","phone":null,"id":"c41418fb-faa4-4772-bd24-ffd2caaed908","first_name":"Test","last_name":"TEST","last_active":"2018-03-08T14:55:49.861Z","last_view_id":"4d8fe22e-67cf-4479-80be-1b3c12468b72","last_project":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","has_seen_fre":true,"user_roles":[{"id":23213,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:24:05.699Z","updated_at":"2017-03-21T12:24:05.699Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23215,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:42:20.179Z","updated_at":"2017-03-21T12:42:20.179Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23439,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-27T11:16:54.046Z","updated_at":"2017-03-27T11:16:54.046Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22906,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:32:32.469Z","updated_at":"2017-03-13T10:32:32.469Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22907,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:35:00.433Z","updated_at":"2017-03-13T11:19:37.328Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32698,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:13:22.121Z","updated_at":"2018-02-14T10:13:22.121Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32701,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:31:26.557Z","updated_at":"2018-02-14T10:31:26.557Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":33468,"role":"PROJECT_ADMIN","XXeated_at":"2018-03-07T15:51:17.005Z","updated_at":"2018-03-07T15:51:17.005Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"}],"projects":[{"project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a"}},{"project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4"}},{"project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef"}},{"project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52"}},{"project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317"}},{"project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34"}},{"project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45"}},{"project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"}}]};gon.projects=[{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":26.22,"trailing_week_ppc":0,"image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":65.79,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":45.41,"trailing_week_ppc":33.33,"image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":72.04,"trailing_week_ppc":23.08,"image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":66.67,"trailing_week_ppc":0.0,"image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":81.8,"trailing_week_ppc":10.34,"image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}];gon.highcharts_path="\/assets\/highcharts-34b694afdf78e0fec0852867d3cfe0f1b54e302677f832887d978180c39ddd99.js";gon.urls={"logout":"\/logout","profile":"https:\/\/accounts.TEST.com","help":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/","community":"http:\/\/forums..com/t5\/bim-360\/ct-p\/2025","ideas":"http:\/\/forums.TEST.com\/t5\/bim-360-ideastation\/idb-p\/2032","support":"https:\/\/sso..com\/idp\/startSSO.ping?PartnerSpId=https:\/\/saml_jit.salesforce.com\u0026TargetResource=%2Fcustomer%2Fapex%\u0026cparam1=00D300000008uIL\u0026cparam2=0DM3A000000PDsL\u0026cparam3=https:\/\/TEST-communities.force.com\/customer\u0026cparam4=Plan","whats_new":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/?guid=GUID-8970143C-3A63-4E6A-855D-585927C8EACD","facebook":"https:\/\/www..com\/","twitter":"https:\/\/.com\/","youtube":"https:\/\/www..com\/user\/","privacy":"http:\/\/www..com\/privacy","terms":"http:\/\/www..com\/termsofservice"};//]]>'

try:
    found = re.findall(r'gon.projects=\[(.+?)]\;gon.highcharts_path', output)
except AttributeError:
    found = ''
    
data_out = ast.literal_eval(found[0])

type(data_out)
Gives me a tuple currently...

Any help would be much appreciated, brand new to Python FYI! Been learning for a week so if i'm missing the obvious please let me know...
Find
Reply
wavic
So-and-so of the Yard
Posts: 2,955
Threads: 48
Joined: Sep 2016
Reputation: 90
#2
Mar-08-2018, 07:03 PM
Where did you get this data? Can you get the data as JSON or xml? This looks like some formatted data but I don't recognize it
"As they say in Mexico 'dosvidaniya'. That makes two vidaniyas."
https://freedns.afraid.org
Find
Reply
ck3mp
Programmer named Tim
Posts: 5
Threads: 1
Joined: Mar 2018
Reputation: 0
#3
Mar-08-2018, 08:41 PM (This post was last modified: Mar-08-2018, 08:41 PM by ck3mp.)
The data comes from a system that I have no control over unfortunately so i'm stuck with it.

I've got it to this:
{"root": [{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"26.22","trailing_week_ppc":"0","image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"65.79","trailing_week_ppc":"0","image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"45.41","trailing_week_ppc":"33.33","image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"72.04","trailing_week_ppc":"23.08","image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"66.67","trailing_week_ppc":"0.0","image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"81.8","trailing_week_ppc":"10.34","image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"0","trailing_week_ppc":"0","image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":"0","trailing_week_ppc":"0","image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}]}
by using this: (this is awful...)
import re
import csv

output = '//<![CDATA[window.gon={};gon.current_user={"email":"[email protected]","phone":null,"id":"c41418fb-faa4-4772-bd24-ffd2caaed908","first_name":"Test","last_name":"TEST","last_active":"2018-03-08T14:55:49.861Z","last_view_id":"4d8fe22e-67cf-4479-80be-1b3c12468b72","last_project":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","has_seen_fre":true,"user_roles":[{"id":23213,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:24:05.699Z","updated_at":"2017-03-21T12:24:05.699Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23215,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:42:20.179Z","updated_at":"2017-03-21T12:42:20.179Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23439,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-27T11:16:54.046Z","updated_at":"2017-03-27T11:16:54.046Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22906,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:32:32.469Z","updated_at":"2017-03-13T10:32:32.469Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22907,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:35:00.433Z","updated_at":"2017-03-13T11:19:37.328Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32698,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:13:22.121Z","updated_at":"2018-02-14T10:13:22.121Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32701,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:31:26.557Z","updated_at":"2018-02-14T10:31:26.557Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":33468,"role":"PROJECT_ADMIN","XXeated_at":"2018-03-07T15:51:17.005Z","updated_at":"2018-03-07T15:51:17.005Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"}],"projects":[{"project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a"}},{"project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4"}},{"project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef"}},{"project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52"}},{"project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317"}},{"project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34"}},{"project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45"}},{"project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"}}]};gon.projects=[{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":26.22,"trailing_week_ppc":0,"image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":65.79,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":45.41,"trailing_week_ppc":33.33,"image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":72.04,"trailing_week_ppc":23.08,"image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":66.67,"trailing_week_ppc":0.0,"image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":81.8,"trailing_week_ppc":10.34,"image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}];gon.highcharts_path="\/assets\/highcharts-34b694afdf78e0fec0852867d3cfe0f1b54e302677f832887d978180c39ddd99.js";gon.urls={"logout":"\/logout","profile":"https:\/\/accounts.TEST.com","help":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/","community":"http:\/\/forums..com/t5\/bim-360\/ct-p\/2025","ideas":"http:\/\/forums.TEST.com\/t5\/bim-360-ideastation\/idb-p\/2032","support":"https:\/\/sso..com\/idp\/startSSO.ping?PartnerSpId=https:\/\/saml_jit.salesforce.com\u0026TargetResource=%2Fcustomer%2Fapex%\u0026cparam1=00D300000008uIL\u0026cparam2=0DM3A000000PDsL\u0026cparam3=https:\/\/TEST-communities.force.com\/customer\u0026cparam4=Plan","whats_new":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/?guid=GUID-8970143C-3A63-4E6A-855D-585927C8EACD","facebook":"https:\/\/www..com\/","twitter":"https:\/\/.com\/","youtube":"https:\/\/www..com\/user\/","privacy":"http:\/\/www..com\/privacy","terms":"http:\/\/www..com\/termsofservice"};//]]>'
 
try:
    found = re.findall(r'gon.projects=\[(.+?)]\;gon.highcharts_path', output)
except AttributeError:
    found = ''
    
test = found[0]

output = test.replace('\"','')
output1 = output.replace(":","\":\"")
output2 = output1.replace(",","\",\"")
output3 = output2.replace('}\",\"{','\"},{\"')
output4 = output3.replace('{id','{\"id')
output5 = output4[:-1]
output6 = "{\"root\": [" + output5 + "\"}]}"

print(output6)

data json.loads(output6)
I was going for converting it to JSON to then parse it
Find
Reply
buran

Super Moderators
Posts: 8,090
Threads: 154
Joined: Sep 2016
Reputation: 582
#4
Mar-08-2018, 08:52 PM
can you provide more information about the website, how you call the website and the full response you get? i.e. we need broader perspective ti your problem.
this looks like Rails gon gem being used to pass data to javascript
https://github.com/gazay/gon
https://www.railscarma.com/blog/technica...ipt-rails/

there might be better approach to access the data
Find
Reply
ck3mp
Programmer named Tim
Posts: 5
Threads: 1
Joined: Mar 2018
Reputation: 0
#5
Mar-08-2018, 09:00 PM
So the website is an application provided to my company by an external provider. It has no API access or any other form of interfacing other than the browser. I realise this is a very messy way to get that data but this is my final solution after 6 months of other approaches.

The data in the content variable in the first post is the innerHTML from a script tag that is scraped by Python and Selenium. The only extra data outside of that string is the <script> </script> tags.
Find
Reply
Gribouillis

Super Moderators
Posts: 4,498
Threads: 69
Joined: Jan 2018
Reputation: 330
#6
Mar-08-2018, 10:32 PM (This post was last modified: Mar-08-2018, 10:35 PM by Gribouillis.)
Python's own tokenizer is able to tokenize the data. It could be a first step to seriously parsing the string
from tokenize import generate_tokens
from io import StringIO
f = StringIO(output[11:-5])
L = list(generate_tokens(f.readline))
for tok in L:
    print(tok.type, tok.string)
Find
Reply
ck3mp
Programmer named Tim
Posts: 5
Threads: 1
Joined: Mar 2018
Reputation: 0
#7
Mar-09-2018, 08:51 AM (This post was last modified: Mar-09-2018, 09:45 AM by ck3mp.)
Thank you for this.

Using this and the regular expression I've got to this now... It seems closer!

"id"
"0c9d534a-5af4-4fa5-8cd7-668432775317"
"name"
"test company UK Sandbox"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"0c9d534a-5af4-4fa5-8cd7-668432775317"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
""
"start_date"
"2015-06-01"
"id"
"3c47d67b-d2b5-494e-8c37-0552dfb1449a"
"name"
"XXMA0069 - TEST"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"3c47d67b-d2b5-494e-8c37-0552dfb1449a"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image"
"start_date"
"2015-10-05"
"id"
"ae326656-0523-4c8f-894d-18d6f615c4d4"
"name"
"XXMA006"
"account"
"test company"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"ae326656-0523-4c8f-894d-18d6f615c4d4"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
""
"start_date"
"2015-10-05"
"id"
"944036a2-0aaf-42bf-851b-9b0ced44df45"
"name"
"XXMA0076 - TEST"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"944036a2-0aaf-42bf-851b-9b0ced44df45"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image"
"start_date"
"2016-10-24"
"id"
"11fa1215-0b4a-4437-9c5e-31dbe9599b52"
"name"
"XXMA0082 - QEHB PFP Project"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"11fa1215-0b4a-4437-9c5e-31dbe9599b52"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
""
"start_date"
"2017-01-03"
"id"
"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"
"name"
"XXXX0536 - Audley Coopers Hill"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
""
"start_date"
"2017-02-10"
"id"
"13cdb397-a45e-4241-a17e-65a9f20235ef"
"name"
"Development - TEST TEST"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"13cdb397-a45e-4241-a17e-65a9f20235ef"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
""
"start_date"
"2017-02-15"
"id"
"2dbd8b6e-a096-4f31-96ca-82662b44dc34"
"name"
"sean0007 - Test"
"account"
"test company - Pilot"
"account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"ea_project_id"
"2dbd8b6e-a096-4f31-96ca-82662b44dc34"
"ea_account_id"
"704fbc55-0565-496f-94e6-2e0c95b98327"
"project_ppc_to_date"
"trailing_week_ppc"
"image_path"
"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image"
"start_date"
"2018-02-02"
from tokenize import generate_tokens
from io import StringIO
import re

output = '//<![CDATA[window.gon={};gon.current_user={"email":"[email protected]","phone":null,"id":"c41418fb-faa4-4772-bd24-ffd2caaed908","first_name":"Test","last_name":"TEST","last_active":"2018-03-08T14:55:49.861Z","last_view_id":"4d8fe22e-67cf-4479-80be-1b3c12468b72","last_project":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","has_seen_fre":true,"user_roles":[{"id":23213,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:24:05.699Z","updated_at":"2017-03-21T12:24:05.699Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23215,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:42:20.179Z","updated_at":"2017-03-21T12:42:20.179Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23439,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-27T11:16:54.046Z","updated_at":"2017-03-27T11:16:54.046Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22906,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:32:32.469Z","updated_at":"2017-03-13T10:32:32.469Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22907,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:35:00.433Z","updated_at":"2017-03-13T11:19:37.328Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32698,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:13:22.121Z","updated_at":"2018-02-14T10:13:22.121Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32701,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:31:26.557Z","updated_at":"2018-02-14T10:31:26.557Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":33468,"role":"PROJECT_ADMIN","XXeated_at":"2018-03-07T15:51:17.005Z","updated_at":"2018-03-07T15:51:17.005Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"}],"projects":[{"project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a"}},{"project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4"}},{"project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef"}},{"project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52"}},{"project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317"}},{"project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34"}},{"project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45"}},{"project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"}}]};gon.projects=[{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":26.22,"trailing_week_ppc":0,"image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":65.79,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":45.41,"trailing_week_ppc":33.33,"image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":72.04,"trailing_week_ppc":23.08,"image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":66.67,"trailing_week_ppc":0.0,"image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":81.8,"trailing_week_ppc":10.34,"image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}];gon.highcharts_path="\/assets\/highcharts-34b694afdf78e0fec0852867d3cfe0f1b54e302677f832887d978180c39ddd99.js";gon.urls={"logout":"\/logout","profile":"https:\/\/accounts.TEST.com","help":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/","community":"http:\/\/forums..com/t5\/bim-360\/ct-p\/2025","ideas":"http:\/\/forums.TEST.com\/t5\/bim-360-ideastation\/idb-p\/2032","support":"https:\/\/sso..com\/idp\/startSSO.ping?PartnerSpId=https:\/\/saml_jit.salesforce.com\u0026TargetResource=%2Fcustomer%2Fapex%\u0026cparam1=00D300000008uIL\u0026cparam2=0DM3A000000PDsL\u0026cparam3=https:\/\/TEST-communities.force.com\/customer\u0026cparam4=Plan","whats_new":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/?guid=GUID-8970143C-3A63-4E6A-855D-585927C8EACD","facebook":"https:\/\/www..com\/","twitter":"https:\/\/.com\/","youtube":"https:\/\/www..com\/user\/","privacy":"http:\/\/www..com\/privacy","terms":"http:\/\/www..com\/termsofservice"};//]]>'
 
try:
    found = re.findall(r'gon.projects=\[(.+?)]\;gon.highcharts_path', output)
except AttributeError:
    found = ''

f = StringIO(found[0])
L = list(generate_tokens(f.readline))
for tok in L:
    if tok.type == 3:
        print(tok.string)
Am I right in thinking if I can pair them up into a dict I can write the dict to the CSV file?
Getting closer... I cant now figure out why it only adds the first project from my list to the dict...

from tokenize import generate_tokens
from io import StringIO
import re

output = '//<![CDATA[window.gon={};gon.current_user={"email":"[email protected]","phone":null,"id":"c41418fb-faa4-4772-bd24-ffd2caaed908","first_name":"Test","last_name":"TEST","last_active":"2018-03-08T14:55:49.861Z","last_view_id":"4d8fe22e-67cf-4479-80be-1b3c12468b72","last_project":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","has_seen_fre":true,"user_roles":[{"id":23213,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:24:05.699Z","updated_at":"2017-03-21T12:24:05.699Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23215,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:42:20.179Z","updated_at":"2017-03-21T12:42:20.179Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23439,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-27T11:16:54.046Z","updated_at":"2017-03-27T11:16:54.046Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22906,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:32:32.469Z","updated_at":"2017-03-13T10:32:32.469Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22907,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:35:00.433Z","updated_at":"2017-03-13T11:19:37.328Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32698,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:13:22.121Z","updated_at":"2018-02-14T10:13:22.121Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32701,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:31:26.557Z","updated_at":"2018-02-14T10:31:26.557Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":33468,"role":"PROJECT_ADMIN","XXeated_at":"2018-03-07T15:51:17.005Z","updated_at":"2018-03-07T15:51:17.005Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"}],"projects":[{"project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a"}},{"project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4"}},{"project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef"}},{"project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52"}},{"project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317"}},{"project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34"}},{"project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45"}},{"project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"}}]};gon.projects=[{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":26.22,"trailing_week_ppc":0,"image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":65.79,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":45.41,"trailing_week_ppc":33.33,"image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":72.04,"trailing_week_ppc":23.08,"image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":66.67,"trailing_week_ppc":0.0,"image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":81.8,"trailing_week_ppc":10.34,"image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}];gon.highcharts_path="\/assets\/highcharts-34b694afdf78e0fec0852867d3cfe0f1b54e302677f832887d978180c39ddd99.js";gon.urls={"logout":"\/logout","profile":"https:\/\/accounts.TEST.com","help":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/","community":"http:\/\/forums..com/t5\/bim-360\/ct-p\/2025","ideas":"http:\/\/forums.TEST.com\/t5\/bim-360-ideastation\/idb-p\/2032","support":"https:\/\/sso..com\/idp\/startSSO.ping?PartnerSpId=https:\/\/saml_jit.salesforce.com\u0026TargetResource=%2Fcustomer%2Fapex%\u0026cparam1=00D300000008uIL\u0026cparam2=0DM3A000000PDsL\u0026cparam3=https:\/\/TEST-communities.force.com\/customer\u0026cparam4=Plan","whats_new":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/?guid=GUID-8970143C-3A63-4E6A-855D-585927C8EACD","facebook":"https:\/\/www..com\/","twitter":"https:\/\/.com\/","youtube":"https:\/\/www..com\/user\/","privacy":"http:\/\/www..com\/privacy","terms":"http:\/\/www..com\/termsofservice"};//]]>'
 
try:
    found = re.findall(r'gon.projects=\[(.+?)]\;gon.highcharts_path', output)
except AttributeError:
    found = ''

f = StringIO(found[0])
L = list(generate_tokens(f.readline))

tester = []
for tok in L:
    if tok.type == 3:
        output = tok.string.replace('\"','')
        tester.append(output)

dict(zip(tester[0:][::2], tester[1:][::2]))
This is the output and is exactly what I need (I think) but I need it for all projects...
{'account': 'test company - Pilot',
 'account_id': '704fbc55-0565-496f-94e6-2e0c95b98327',
 'ea_account_id': '704fbc55-0565-496f-94e6-2e0c95b98327',
 'ea_project_id': '2dbd8b6e-a096-4f31-96ca-82662b44dc34',
 'id': '2dbd8b6e-a096-4f31-96ca-82662b44dc34',
 'image_path': '\\/api\\/v1\\/projects\\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\\/image',
 'name': 'sean0007 - Test',
 'project_ppc_to_date': 'trailing_week_ppc',
 'start_date': '2018-02-02'}
if I run:

print(tester[0:][::2])
print(tester[1:][::2])
I get all the keys and values perfectly... but I cant get them into a dict...
Find
Reply
ck3mp
Programmer named Tim
Posts: 5
Threads: 1
Joined: Mar 2018
Reputation: 0
#8
Mar-09-2018, 12:29 PM
Solved: (Open to code improvements!)

from tokenize import generate_tokens
from io import StringIO
import re
import numpy as np

output = '//<![CDATA[window.gon={};gon.current_user={"email":"[email protected]","phone":null,"id":"c41418fb-faa4-4772-bd24-ffd2caaed908","first_name":"Test","last_name":"TEST","last_active":"2018-03-08T14:55:49.861Z","last_view_id":"4d8fe22e-67cf-4479-80be-1b3c12468b72","last_project":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","has_seen_fre":true,"user_roles":[{"id":23213,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:24:05.699Z","updated_at":"2017-03-21T12:24:05.699Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23215,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-21T12:42:20.179Z","updated_at":"2017-03-21T12:42:20.179Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":23439,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-27T11:16:54.046Z","updated_at":"2017-03-27T11:16:54.046Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22906,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:32:32.469Z","updated_at":"2017-03-13T10:32:32.469Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":22907,"role":"PROJECT_ADMIN","XXeated_at":"2017-03-13T10:35:00.433Z","updated_at":"2017-03-13T11:19:37.328Z","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32698,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:13:22.121Z","updated_at":"2018-02-14T10:13:22.121Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":32701,"role":"PROJECT_ADMIN","XXeated_at":"2018-02-14T10:31:26.557Z","updated_at":"2018-02-14T10:31:26.557Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"},{"id":33468,"role":"PROJECT_ADMIN","XXeated_at":"2018-03-07T15:51:17.005Z","updated_at":"2018-03-07T15:51:17.005Z","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","default_view_setting_id":null,"status":"active"}],"projects":[{"project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a"}},{"project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4"}},{"project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef"}},{"project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52"}},{"project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","role":"PROJECT_ADMIN","company_id":"b7a0d950-a6fe-4c84-9a76-79eb43cae49f","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317"}},{"project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34"}},{"project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45"}},{"project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","role":"PROJECT_ADMIN","company_id":"4319a85d-7545-4734-a127-7db9efa4b329","notification_setting":{"instant_email":false,"daily_email":false,"user_id":"c41418fb-faa4-4772-bd24-ffd2caaed908","project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7"}}]};gon.projects=[{"id":"0c9d534a-5af4-4fa5-8cd7-668432775317","name":"test company UK Sandbox","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"0c9d534a-5af4-4fa5-8cd7-668432775317","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":26.22,"trailing_week_ppc":0,"image_path":"","start_date":"2015-06-01"},{"id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","name":"XXMA0069 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"3c47d67b-d2b5-494e-8c37-0552dfb1449a","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":65.79,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/3c47d67b-d2b5-494e-8c37-0552dfb1449a\/image","start_date":"2015-10-05"},{"id":"ae326656-0523-4c8f-894d-18d6f615c4d4","name":"XXMA006","account":"test company","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"ae326656-0523-4c8f-894d-18d6f615c4d4","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":45.41,"trailing_week_ppc":33.33,"image_path":"","start_date":"2015-10-05"},{"id":"944036a2-0aaf-42bf-851b-9b0ced44df45","name":"XXMA0076 - TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"944036a2-0aaf-42bf-851b-9b0ced44df45","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":72.04,"trailing_week_ppc":23.08,"image_path":"\/api\/v1\/projects\/944036a2-0aaf-42bf-851b-9b0ced44df45\/image","start_date":"2016-10-24"},{"id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","name":"XXMA0082 - QEHB PFP Project","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"11fa1215-0b4a-4437-9c5e-31dbe9599b52","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":66.67,"trailing_week_ppc":0.0,"image_path":"","start_date":"2017-01-03"},{"id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","name":"XXXX0536 - Audley Coopers Hill","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"da199d7f-b145-4c5f-89ec-3f9dcacf98d7","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":81.8,"trailing_week_ppc":10.34,"image_path":"","start_date":"2017-02-10"},{"id":"13cdb397-a45e-4241-a17e-65a9f20235ef","name":"Development - TEST TEST","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"13cdb397-a45e-4241-a17e-65a9f20235ef","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"","start_date":"2017-02-15"},{"id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","name":"sean0007 - Test","account":"test company - Pilot","account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","ea_project_id":"2dbd8b6e-a096-4f31-96ca-82662b44dc34","ea_account_id":"704fbc55-0565-496f-94e6-2e0c95b98327","project_ppc_to_date":0,"trailing_week_ppc":0,"image_path":"\/api\/v1\/projects\/2dbd8b6e-a096-4f31-96ca-82662b44dc34\/image","start_date":"2018-02-02"}];gon.highcharts_path="\/assets\/highcharts-34b694afdf78e0fec0852867d3cfe0f1b54e302677f832887d978180c39ddd99.js";gon.urls={"logout":"\/logout","profile":"https:\/\/accounts.TEST.com","help":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/","community":"http:\/\/forums..com/t5\/bim-360\/ct-p\/2025","ideas":"http:\/\/forums.TEST.com\/t5\/bim-360-ideastation\/idb-p\/2032","support":"https:\/\/sso..com\/idp\/startSSO.ping?PartnerSpId=https:\/\/saml_jit.salesforce.com\u0026TargetResource=%2Fcustomer%2Fapex%\u0026cparam1=00D300000008uIL\u0026cparam2=0DM3A000000PDsL\u0026cparam3=https:\/\/TEST-communities.force.com\/customer\u0026cparam4=Plan","whats_new":"http:\/\/help.TEST.com\/view\/BIM360P\/ENU\/?guid=GUID-8970143C-3A63-4E6A-855D-585927C8EACD","facebook":"https:\/\/www..com\/","twitter":"https:\/\/.com\/","youtube":"https:\/\/www..com\/user\/","privacy":"http:\/\/www..com\/privacy","terms":"http:\/\/www..com\/termsofservice"};//]]>'

try:
    found = re.findall(r'gon.projects=\[(.+?)]\;gon.highcharts_path', output)
except AttributeError:
    found = ''

f = StringIO(found[0])
L = list(generate_tokens(f.readline))

list1 = []
for tok in L:
    if (tok.type == 3) or (tok.type == 2):
        output = tok.string.replace('\"\"', '\"Blank\"')
        list1.append(output)

f = open('csvfile.csv','w')
keys = list1[0:][::2]
seen = set()
seen_add = seen.add
keysu= [x for x in keys if not (x in seen or seen_add(x))]
values = list1[1:][::2]
f.write(','.join(keysu)+'\n')
a = np.array(values).reshape(int(len(values)/len(keysu)),len(keysu))
for i in a:
    f.write(','.join(i)+'\n')
Find
Reply
nilamo
Last Thursdayist
Super Moderators
Posts: 3,458
Threads: 101
Joined: Sep 2016
Reputation: 143
#9
Apr-17-2018, 07:39 PM
(Mar-09-2018, 12:29 PM)ck3mp Wrote: (Open to code improvements!)
Close the file you open when you're done with it :p
Find
Reply
Larz60+
aetate et sapientia
Super Moderators
Posts: 11,871
Threads: 474
Joined: Sep 2016
Reputation: 452
#10
Apr-17-2018, 08:08 PM
it's better to use 'with' on file IO like:
with open('csvfile.csv','w') as f:
    keys = list1[0:][::2]
    seen = set()
    seen_add = seen.add
    keysu = [x for x in keys if not (x in seen or seen_add(x))]
    values = list1[1:][::2]
    f.write(','.join(keysu) + '\n')
    a = np.array(values).reshape(int(len(values) / len(keysu)), len(keysu))
    for i in a:
        f.write(','.join(i) + '\n')
Then you don't have to worry about closing the file, as it is automatically done for you.
Find
Reply


Forum Jump:

User Panel Messages

Log Out
My Profile
Pay your profile a visit
User Control Panel
Do some changes on your profile
My Messages
View private messages unread
Avatar
Change avatar
Signature
Change signature
Announcements
Announcement #1 8/1/2020
Announcement #2 8/2/2020
Announcement #3 8/6/2020