prima versione

.gitignore

@@ -0,0 +1,155 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+config.py
+bollette.csv

agsm.py

@@ -0,0 +1,318 @@
+import requests
+import re
+import csv
+import fitz
+import re
+import pysftp
+import os
+import time
+import paho.mqtt.client as paho
+from selenium import webdriver
+import subprocess
+import platform
+from pyzabbix import ZabbixMetric, ZabbixSender
+import logging
+import logging.handlers
+from pathlib import Path
+import config
+
+# python -m venv env
+# attivare ambiente
+# pip install -r requirements.txt
+
+#TODO aggiungere autocreazione di bollette.csv se non esiste, con questi campi: data,fattura,luogo,gas,elettricità,totale
+
+
+versione = '0.1'
+#logging, max 2M a file e ne tengo solo 5
+LOG_FILENAME = Path('agsm.log')
+logger = logging.getLogger('agsmlog')
+logger.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+handler = logging.handlers.RotatingFileHandler(
+            LOG_FILENAME, maxBytes=2097152, backupCount=5)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+
+
+mqttbroker=config.mqttbroker
+mqttport=config.mqttport
+
+login = config.login
+passwd = config.passwd
+
+loginacq = config.loginacq
+passwdacq = config.passwdacq
+
+
+loginds = config.loginds
+passds = config.passds
+
+diskstation = config.diskstation
+
+url = 'https://countbox.agsm.it/'
+
+
+
+cnopts = pysftp.CnOpts()
+cnopts.hostkeys = None 
+
+def bollettaGasEnergia(dati):
+    r = re.compile('Fattura n° *.*')
+    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
+    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
+    data=data[0]
+    fattura=fattura[0]
+
+    r = re.compile('energia in *.*')
+    dove = ([x.split()[4:5] for x in r.findall(dati)])
+    dove=dove[0][0]
+
+    r = re.compile('TV: € [0-9]+,[0-9]+')
+    elettricita=([x.split()[-1] for x in r.findall(dati)])
+    elettricita = elettricita[0]
+    elettricita = elettricita.replace(',','.')
+
+    client1= paho.Client("control1")                           #create client object
+    client1.on_publish = on_publish                          #assign function to callback
+    client1.connect(mqttbroker,mqttport)                                 #establish connection
+    client1.publish('bollette/'+ dove.lower() +'/elettricita', elettricita, qos=0,retain=True)                  #publish
+
+
+    r = re.compile('NATURALE: € [0-9]+,[0-9]+')
+    gas=([x.split()[-1] for x in r.findall(dati)])
+    gas = gas[0]
+    gas = gas.replace(',','.')
+
+    client1= paho.Client("control1")                           #create client object
+    client1.on_publish = on_publish                          #assign function to callback
+    client1.connect(mqttbroker,mqttport)                                 #establish connection
+    client1.publish('bollette/'+ dove.lower() +'/gas', gas, qos=0,retain=True)                 #publish
+
+
+    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
+    tot=([x.split()[-1] for x in r.findall(dati)])
+    tot = tot[0]
+    tot = tot.replace(',','.')
+
+    print('Lettura data: {} n: {} dove: {} importo gas: {}€ importo elettricità: {}€ Totale: {}€'.format(data,fattura,dove,gas,elettricita,tot))
+    
+    f = open("bollette.csv", "a")
+    f.write(data + ',' + fattura + ',' + dove.lower() + ',' + gas + ',' + elettricita + ',' + tot + '\n')
+    f.close()
+
+def bollettaGas(dati):
+    r = re.compile('Fattura n° *.*')
+    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
+    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
+    data=data[0]
+    fattura=fattura[0]
+
+    r = re.compile('fornendo gas in *.*')
+    dove = ([x.split()[4:5] for x in r.findall(dati)])
+    dove=dove[0][0]
+
+    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
+    gas=([x.split()[-1] for x in r.findall(dati)])
+    gas = gas[0]
+    gas = gas.replace(',','.')
+
+    client1= paho.Client("control1")                           #create client object
+    client1.on_publish = on_publish                          #assign function to callback
+    client1.connect(mqttbroker,mqttport)                                 #establish connection
+    client1.publish('bollette/'+ dove.lower() +'/gas', gas, qos=0,retain=True)                   #publish
+
+
+    print('Lettura data: {} n: {} dove: {} importo gas: {}€ '.format(data,fattura,dove,gas))
+    f = open("bollette.csv", "a")
+    f.write(data + ',' + fattura + ',' + dove + ',' + gas + ',,' + gas + '\n')
+    f.close()
+
+def bollettaElettrica(dati):
+    r = re.compile('Fattura n° *.*')
+    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
+    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
+    data=data[0]
+    fattura=fattura[0]
+
+    r = re.compile('fornendo energia in *.*')
+    dove = ([x.split()[4:5] for x in r.findall(dati)])
+    dove=dove[0][0]
+
+    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
+    elettricita=([x.split()[-1] for x in r.findall(dati)])
+    elettricita = elettricita[0]
+    elettricita = elettricita.replace(',','.')
+
+    client1= paho.Client("control1")                           #create client object
+    client1.on_publish = on_publish                          #assign function to callback
+    client1.connect(mqttbroker,mqttport)                                 #establish connection
+    client1.publish('bollette/'+ dove.lower() +'/elettricita', elettricita, qos=0,retain=True)                  #publish
+
+
+    print('Lettura data: {} n: {} dove: {} importo elettricità: {}€ '.format(data,fattura,dove,elettricita))
+    f = open("bollette.csv", "a")
+    f.write(data + ',' + fattura + ',' + dove + ',,' + elettricita + ',' + elettricita + '\n')
+    f.close()
+
+def on_publish(client,userdata,result):             #create function for callback
+    print("data published \n")
+    pass
+
+def agsmenergia():
+    giaScaricati = [] #doc che si trovano gia in csv.
+    with open('bollette.csv') as csv_file:
+            csv_reader = csv.reader(csv_file, delimiter=',')
+            line_count = 0
+            for row in csv_reader:
+                if line_count == 0:
+                    line_count += 1
+                else:
+                    try:
+                        a = row[1]
+                        a=a.split('/')
+                        giaScaricati.append(a[1])
+                    except: #mai scaricato nulla
+                        pass
+
+    # Fill in your details here to be posted to the login form.
+    payload = {
+        'username': login,
+        'password': passwd
+    }
+    # Use 'with' to ensure the session context is closed after use.
+    with requests.Session() as s:
+        p = s.post(url, data=payload)
+        # print the html returned or something more intelligent to see if it's a successful login page.
+        #print (p.text)
+        try:
+            page = s.get('https://countbox.agsm.it/bills')
+            html = page.text 
+            r = re.compile('<a href="/bills/download/[0-9]+/[0-9]+/ita')
+            dlraw=r.findall(html)
+            zabbix_send('logincountbox',1)
+        except:
+            logger.error('Problemi di connessione al sito countbox')
+            zabbix_send('logincountbox',0)
+
+        for a in dlraw:
+            link = a.replace('<a href="','')
+            nome = link.split('/')
+            if len(nome[4]) <= 6:
+                nome = '0' + nome[4]
+            else:
+                nome = nome[4]
+            try:
+                if nome not in giaScaricati:
+                    link = 'https://countbox.agsm.it' + link
+                    print(link)
+                    dl = s.get(link, allow_redirects=True)
+                    pdffile = 'bolletta-' + nome + '.pdf'
+                    f = open(pdffile, 'wb').write(dl.content)
+
+
+                    doc = fitz.open(pdffile)
+                    page = doc.load_page(0)
+                    dati = page.get_text('text')
+                
+
+                    r = re.compile('BOLLETTA PER LA FORNITURA DI GAS NATURALE')
+                    q = re.compile('BOLLETTA PER LA FORNITURA DI ENERGIA ELETTRICA E GAS NATURALE')
+
+
+                    if len(r.findall(dati)) == 1:
+                        bollettaGas(dati)
+                    elif len(q.findall(dati)) == 1:
+                        bollettaGasEnergia(dati)
+                    else:
+                        bollettaElettrica(dati)
+
+                    try:
+                        with pysftp.Connection(host=diskstation,username=loginds,password=passds,cnopts=cnopts) as sftp:
+                            sftp.cwd('/Romito/Casa/bollette')
+                            sftp.put(pdffile, '/Romito/Casa/bollette/' + pdffile)
+                            sftp.put('bollette.csv', '/Romito/Casa/bollette/bollette.csv')
+                            time.sleep(5)
+                        doc.close()
+                        os.remove(pdffile)
+                        zabbix_send('sftpmove',1)
+                    except:
+                        logger.error('Invio sftp o spostamento file fallito')
+                        zabbix_send('sftpmove',0)
+                logger.info('Agsmenergia ok')
+                zabbix_send('agsmenergia',1)
+            except:
+                logger.error('Qualcosaè  andato storto con agsmenergia')
+                zabbix_send('agsmenergia',0)
+
+def acqueveronesi():
+    if platform.node() == 'ds7':
+        try:
+            subprocess.run(["docker","start", "SeleniumGrid"])
+            time.sleep(30)
+            zabbix_send('dockerstart',1)
+        except:
+            logger.error('Avvio docker fallito')
+            zabbix_send('dockerstart',0)
+    
+    try:
+        firefox_options = webdriver.FirefoxOptions()
+        driver = webdriver.Remote(
+            command_executor='http://ds7.romito.net:4444',
+            options=firefox_options
+        )
+        driver.set_page_load_timeout(10)
+        driver.get("https://acqueveronesi.cloudeng.it/")
+        
+        driver.find_element_by_id('commons-alert-close').click()
+        driver.find_element_by_id('username').send_keys(loginacq)
+        driver.find_element_by_id ('password').send_keys(passwdacq)
+        driver.find_element_by_id('submit-web').click()
+        driver.get('https://acqueveronesi.cloudeng.it/public/netaweb2a/supply/payments?fornituraCod=98071242') #euclide
+        # prontoweb_payment_div-1_div-1_table-1_td-6 euro...
+        '''    
+        importi = driver.find_elements_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]')
+        lista_importi_euclide = []
+        for p in range(len(importi)):
+            lista_importi_euclide.append(importi[p].text)
+        
+        print(lista_importi_euclide)
+        '''
+        #solo il primo della lista... in pratica l'ultima bolletta emessa...
+        importo = driver.find_element_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]').text
+        importo = importo.replace('€. ','')
+        print('Euclide', importo)
+
+        client1= paho.Client("control1")                           #create client object
+        client1.on_publish = on_publish                          #assign function to callback
+        client1.connect(mqttbroker,mqttport)                                 #establish connection
+        client1.publish('bollette/euclide/acqua', importo, qos=0,retain=True)                  #publish
+
+        driver.get('https://acqueveronesi.cloudeng.it/public/netaweb2a/supply/payments?fornituraCod=117801547') #sacchi
+        
+        importo = driver.find_element_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]').text
+        importo = importo.replace('€. ','')
+        print('Sacchi', importo)
+        client1.publish('bollette/sacchi/acqua', importo, qos=0,retain=True)                  #publish
+        driver.quit()
+        if platform.node() == 'ds7':
+            subprocess.run(["docker","stop", "SeleniumGrid"])
+        zabbix_send('acqueveronesi',1)
+    except:
+        logger.error('Qualcosa è andato storto con acque veronesi')
+        zabbix_send('acqueveronesi',0)
+
+
+def zabbix_send(item,dato):
+    # Send metrics to zabbix trapper
+    packet = [
+        ZabbixMetric('bollette', item, dato),
+            ]
+
+    result = ZabbixSender(zabbix_server='zabbixsrv.romito.net', zabbix_port=10051, use_config=None, chunk_size=250, socket_wrapper=None, timeout=10).send(packet)
+
+
+
+agsmenergia()
+acqueveronesi()

config-orig.py

@@ -0,0 +1,16 @@
+#rinominare in config.py
+
+mqttbroker='dns o ip broker'
+mqttport='porta broker'
+
+login = 'login agsm'
+passwd = 'password agsm'
+
+loginds = 'login sftp server'
+passds = 'password  sftp server'
+
+
+loginacq = 'login acque veronesi'
+passwdacq = 'password acque veronesi'
+
+diskstation = 'indirizzo sftp server'