agsm/agsm.py

import requests
import re
import csv
import fitz
import re
import pysftp
import os
import time
import paho.mqtt.client as paho
from selenium import webdriver
import subprocess
import platform
from pyzabbix import ZabbixMetric, ZabbixSender
import logging
import logging.handlers
from pathlib import Path
import config

# python -m venv env
# attivare ambiente
# pip install -r requirements.txt

#TODO aggiungere autocreazione di bollette.csv se non esiste, con questi campi: data,fattura,luogo,gas,elettricità,totale


versione = '0.1'
#logging, max 2M a file e ne tengo solo 5
LOG_FILENAME = Path('agsm.log')
logger = logging.getLogger('agsmlog')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler = logging.handlers.RotatingFileHandler(
            LOG_FILENAME, maxBytes=2097152, backupCount=5)
handler.setFormatter(formatter)
logger.addHandler(handler)



mqttbroker=config.mqttbroker
mqttport=config.mqttport

login = config.login
passwd = config.passwd

loginacq = config.loginacq
passwdacq = config.passwdacq


loginds = config.loginds
passds = config.passds

loginw3 = config.loginw3
passwdw3 = config.passwdw3

diskstation = config.diskstation

url = 'https://countbox.agsm.it/'



cnopts = pysftp.CnOpts()
cnopts.hostkeys = None 

def bollettaGasEnergia(dati):
    r = re.compile('Fattura n° *.*')
    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
    data=data[0]
    fattura=fattura[0]

    r = re.compile('energia in *.*')
    dove = ([x.split()[4:5] for x in r.findall(dati)])
    dove=dove[0][0]

    r = re.compile('TV: € [0-9]+,[0-9]+')
    elettricita=([x.split()[-1] for x in r.findall(dati)])
    elettricita = elettricita[0]
    elettricita = elettricita.replace(',','.')

    client1= paho.Client("control1")                           #create client object
    client1.on_publish = on_publish                          #assign function to callback
    client1.connect(mqttbroker,mqttport)                                 #establish connection
    client1.publish('bollette/'+ dove.lower() +'/elettricita', elettricita, qos=0,retain=True)                  #publish


    r = re.compile('NATURALE: € [0-9]+,[0-9]+')
    gas=([x.split()[-1] for x in r.findall(dati)])
    gas = gas[0]
    gas = gas.replace(',','.')

    client1= paho.Client("control1")                           #create client object
    client1.on_publish = on_publish                          #assign function to callback
    client1.connect(mqttbroker,mqttport)                                 #establish connection
    client1.publish('bollette/'+ dove.lower() +'/gas', gas, qos=0,retain=True)                 #publish


    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
    tot=([x.split()[-1] for x in r.findall(dati)])
    tot = tot[0]
    tot = tot.replace(',','.')

    print('Lettura data: {} n: {} dove: {} importo gas: {}€ importo elettricità: {}€ Totale: {}€'.format(data,fattura,dove,gas,elettricita,tot))
    
    f = open("bollette.csv", "a")
    f.write(data + ',' + fattura + ',' + dove.lower() + ',' + gas + ',' + elettricita + ',' + tot + '\n')
    f.close()

def bollettaGas(dati):
    r = re.compile('Fattura n° *.*')
    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
    data=data[0]
    fattura=fattura[0]

    r = re.compile('fornendo gas in *.*')
    dove = ([x.split()[4:5] for x in r.findall(dati)])
    dove=dove[0][0]

    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
    gas=([x.split()[-1] for x in r.findall(dati)])
    gas = gas[0]
    gas = gas.replace(',','.')

    client1= paho.Client("control1")                           #create client object
    client1.on_publish = on_publish                          #assign function to callback
    client1.connect(mqttbroker,mqttport)                                 #establish connection
    client1.publish('bollette/'+ dove.lower() +'/gas', gas, qos=0,retain=True)                   #publish


    print('Lettura data: {} n: {} dove: {} importo gas: {}€ '.format(data,fattura,dove,gas))
    f = open("bollette.csv", "a")
    f.write(data + ',' + fattura + ',' + dove + ',' + gas + ',,' + gas + '\n')
    f.close()

def bollettaElettrica(dati):
    r = re.compile('Fattura n° *.*')
    data = ([x.split()[-1] for x in r.findall(dati)]) #data fattura
    fattura = ([x.split()[-3] for x in r.findall(dati)]) #n. fattura
    data=data[0]
    fattura=fattura[0]

    r = re.compile('fornendo energia in *.*')
    dove = ([x.split()[4:5] for x in r.findall(dati)])
    dove=dove[0][0]

    r = re.compile('TOTALE DA PAGARE: euro [0-9]+,[0-9]+')
    elettricita=([x.split()[-1] for x in r.findall(dati)])
    elettricita = elettricita[0]
    elettricita = elettricita.replace(',','.')

    client1= paho.Client("control1")                           #create client object
    client1.on_publish = on_publish                          #assign function to callback
    client1.connect(mqttbroker,mqttport)                                 #establish connection
    client1.publish('bollette/'+ dove.lower() +'/elettricita', elettricita, qos=0,retain=True)                  #publish


    print('Lettura data: {} n: {} dove: {} importo elettricità: {}€ '.format(data,fattura,dove,elettricita))
    f = open("bollette.csv", "a")
    f.write(data + ',' + fattura + ',' + dove + ',,' + elettricita + ',' + elettricita + '\n')
    f.close()

def on_publish(client,userdata,result):             #create function for callback
    print("data published \n")
    pass

def agsmenergia():
    giaScaricati = [] #doc che si trovano gia in csv.
    with open('bollette.csv') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',')
            line_count = 0
            for row in csv_reader:
                if line_count == 0:
                    line_count += 1
                else:
                    try:
                        a = row[1]
                        a=a.split('/')
                        giaScaricati.append(a[1])
                    except: #mai scaricato nulla
                        pass

    # Fill in your details here to be posted to the login form.
    payload = {
        'username': login,
        'password': passwd
    }
    # Use 'with' to ensure the session context is closed after use.
    with requests.Session() as s:
        p = s.post(url, data=payload)
        # print the html returned or something more intelligent to see if it's a successful login page.
        #print (p.text)
        try:
            page = s.get('https://countbox.agsm.it/bills')
            html = page.text 
            r = re.compile('<a href="/bills/download/[0-9]+/[0-9]+/ita')
            dlraw=r.findall(html)
            zabbix_send('logincountbox',1)
        except:
            logger.error('Problemi di connessione al sito countbox')
            zabbix_send('logincountbox',0)

        for a in dlraw:
            link = a.replace('<a href="','')
            nome = link.split('/')
            if len(nome[4]) <= 6:
                nome = '0' + nome[4]
            else:
                nome = nome[4]
            try:
                if nome not in giaScaricati:
                    link = 'https://countbox.agsm.it' + link
                    print(link)
                    dl = s.get(link, allow_redirects=True)
                    pdffile = 'bolletta-' + nome + '.pdf'
                    f = open(pdffile, 'wb').write(dl.content)

                    
                    doc = fitz.open(pdffile)
                    pagine = doc.page_count
                    print(pagine)
                    for pagina in range(pagine):
                        
                        dati = doc.load_page(pagina).get_text('text')

                        r = re.compile('1000010000652791') # gas via euclide
                        q = re.compile('1000010000861860') # gas ed elettricità via sacchi
                        t = re.compile('1000010000652788') # elettricità euclide

                        if len(r.findall(dati)) == 1:
                            bollettaGas(dati)
                        if len(q.findall(dati)) == 1:
                            bollettaGasEnergia(dati)
                        if len(t.findall(dati)) == 1:
                            bollettaElettrica(dati)

                    try:
                        with pysftp.Connection(host=diskstation,username=loginds,password=passds,cnopts=cnopts) as sftp:
                            sftp.cwd('/Romito/Casa/bollette')
                            sftp.put(pdffile, '/Romito/Casa/bollette/' + pdffile)
                            sftp.put('bollette.csv', '/Romito/Casa/bollette/bollette.csv')
                            time.sleep(5)
                        doc.close()
                        os.remove(pdffile)
                        zabbix_send('sftpmove',1)
                    except:
                        logger.error('Invio sftp o spostamento file fallito')
                        zabbix_send('sftpmove',0)
                logger.info('Agsmenergia ok')
                zabbix_send('agsmenergia',1)
            
            except:
                logger.error('Qualcosa è  andato storto con agsmenergia')
                zabbix_send('agsmenergia',0)
            

def acqueveronesi():
  
    try:
        firefox_options = webdriver.FirefoxOptions()
        driver = webdriver.Remote(
            command_executor='http://ds7.romito.net:4444',
            options=firefox_options
        )
        driver.set_page_load_timeout(10)
        driver.get("https://acqueveronesi.cloudeng.it/")
        
        driver.find_element_by_id('commons-alert-close').click()
        driver.find_element_by_id('username').send_keys(loginacq)
        driver.find_element_by_id ('password').send_keys(passwdacq)
        driver.find_element_by_id('submit-web').click()
        driver.get('https://acqueveronesi.cloudeng.it/public/netaweb2a/supply/payments?fornituraCod=98071242') #euclide
        # prontoweb_payment_div-1_div-1_table-1_td-6 euro...
        '''    
        importi = driver.find_elements_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]')
        lista_importi_euclide = []
        for p in range(len(importi)):
            lista_importi_euclide.append(importi[p].text)
        
        print(lista_importi_euclide)
        '''
        #solo il primo della lista... in pratica l'ultima bolletta emessa...
        importo = driver.find_element_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]').text
        importo = importo.replace('€. ','')
        print('Euclide', importo)

        client1= paho.Client("control1")                           #create client object
        client1.on_publish = on_publish                          #assign function to callback
        client1.connect(mqttbroker,mqttport)                                 #establish connection
        client1.publish('bollette/euclide/acqua', importo, qos=0,retain=True)                  #publish

        driver.get('https://acqueveronesi.cloudeng.it/public/netaweb2a/supply/payments?fornituraCod=117801547') #sacchi
        
        importo = driver.find_element_by_xpath('//td[@class="prontoweb_payment_div-1_div-1_table-1_td-6"]').text
        importo = importo.replace('€. ','')
        print('Sacchi', importo)
        client1.publish('bollette/sacchi/acqua', importo, qos=0,retain=True)                  #publish
        zabbix_send('acqueveronesi',1)
    except:
        logger.error('Qualcosa è andato storto con acque veronesi')
        zabbix_send('acqueveronesi',0)

def windtre():
    url = 'https://areaclienti.windtre.it/login'
    urlconto = 'https://areaclienti.windtre.it/phx/fixed/conto'

    try:
        firefox_options = webdriver.FirefoxOptions()
        driver = webdriver.Remote(
            command_executor='http://ds7.romito.net:4444',
            options=firefox_options
        )
        driver.set_page_load_timeout(10)
        driver.get(url)
        time.sleep(10)
        try:
            driver.find_element_by_id('close').click()
            print('accetto cookie!')
            
        except:
            pass

        driver.find_element_by_id('username').send_keys(loginw3)
        driver.find_element_by_id ('password').send_keys(passwdw3)
        driver.find_element_by_class_name('btn-login').click()

        time.sleep(10)

        driver.get(urlconto)
        time.sleep(10)
        l = driver.find_element_by_class_name('invoice-table')

        importo = re.search(r"[0-9][0-9],[0-9][0-9]", l.text).group()

        print(importo)
        client1= paho.Client("control1")                           #create client object
        client1.on_publish = on_publish                          #assign function to callback
        client1.connect(mqttbroker,mqttport)                                 #establish connection
        client1.publish('bollette/sacchi/windtre', importo, qos=0,retain=True) 
        zabbix_send('windtre',1)
    except:
        zabbix_send('windtre',0)
    driver.quit()

def zabbix_send(item,dato):
    # Send metrics to zabbix trapper
    packet = [
        ZabbixMetric('bollette', item, dato),
            ]

    result = ZabbixSender(zabbix_server='zabbixsrv.romito.net', zabbix_port=10051, use_config=None, chunk_size=250, socket_wrapper=None, timeout=10).send(packet)



agsmenergia()

if platform.node() == 'ds7':
    try:
        subprocess.run(["docker","start", "SeleniumGrid"])
        time.sleep(30)
        zabbix_send('dockerstart',1)
    except:
        logger.error('Avvio docker fallito')
        zabbix_send('dockerstart',0)

acqueveronesi()
windtre()

if platform.node() == 'ds7':
    subprocess.run(["docker","stop", "SeleniumGrid"])