mitmproxy dziwne zachowanie skryptu - gubi dane

0

Rozwinąłem trochę plik przykładowy z https://github.com/mitmproxy/mitmproxy/blob/master/examples/stub.py żeby przechwytywać komunikaty AMF z ofertami handlowymi w pewnej grze online.

Wszystko działa idealnie dopóki wykonuje tylko elementarne operacje, jak informowanie o przechwyceniu pytania o oferty, przechwyceniu samych ofert, policzeniu ich ilości, czy przeczytania z wiadomości AMF jej numeru sekwencyjnego. chody zaczynają się gdy próbuje wykonywać bardziej czasochłonne czynności - w poniższym kodzie wyhaszowane są fragmenty wrzucające oferty do bazy SQLite - a to dla tego, że przy próbie ich użycia do bazy trafia tylko niewielka część ofert a reszta gdzieś ginie!?

Kto coś może podpowiedzieć?

import sqlite3
import os, time, sys, gzip, fileinput, StringIO
from amfast import decode, decoder, encode, encoder, remoting, class_def, context, logger


def start(ctx):
    """
        Called once on script startup, before any other events.
    """
    ctx.log("start")
    print("start")
#    db = sqlite3.connect(":memory:")
#    dbc = db.cursor()
#    dbc.execute("""
#                create table offki (
#                        myid                    integer primary key autoincrement,
#                        coolDownTime            integer,
#                        created                 integer not null unique on conflict fail,
#                        deleted                 integer,
#                        id                      integer,
#                        isTradeCancled          boolean,
#                        lotsRemaining           integer,
#                        offer                   string not null,
#                        offerAcceptedID         integer,
#                        receiverID              integer,
#                        remainingTime           integer,
#                        removed                 integer,
#                        senderID                integer not null,
#                        senderName              string not null,
#                        slotPos                 integer,
#                        slotType                integer,
#                        type                    integer);
#               """)


def clientconnect(ctx, client_connect):
    """
        Called when a client initiates a connection to the proxy. Note that a
        connection can correspond to multiple HTTP requests
    """
    ctx.log("clientconnect")
    print("clientconnect")

def request(ctx, flow):
    """
        Called when a client request has been received.
    """
    ctx.log("request")
    if flow.request.headers["content-type"] == ["application/x-amf"] and flow.request.content.find('GetAvailableOffers') <> -1:
	rqn=flow.request.content[16:].split('\x00\x00')[0]
	print("## ---> przechwycono zapytanie o oferty, w wiadomosci AMF numer: " + rqn.__str__())

def response(ctx, flow):
    """
       Called when a server response has been received.
    """
    ctx.log("response")
    if flow.response.headers["content-type"] == ["application/x-amf"] and flow.response.content.find('dTradeWindowResultVO') <> -1:
	rsn = flow.response.content[10:].split('\x2f')[0]
	mess = decoder.decode_packet(flow.response.content)
	mess = mess.messages[0].body
	mess = mess.body
	mess = mess['data']
	mess = mess['data']
	mess = mess['tradeOffers']
	
	### PONIzSZE dziala bez zarzutu wypisujac cala (KOMPLETNA!) liste na ekran:
	# print(mess)
	
	###  PONIzSZE dziala bez zarzutu podajac numer wiadomosci AMF oraz laczna liczbe ofert handlowych: 
	print("## ---> oferty przechwycone w odpowiedzi AMF numer: "+rsn.__str__()+"  ilosc wszystkich ofert: " + len(mess).__str__())

### 



#	print(mess)
#	q = """ INSERT INTO offki (coolDownTime, created, deleted, id, isTradeCancled, lotsRemaining, offer, offerAcceptedID,
#                                                     receiverID, remainingTime, removed, senderID, senderName, slotPos, slotType, type)
#                VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
#            """
#	oflist=[]
#	for i in range(len(mess)):
#		oflist.append(int(mess[i]['coolDownTime']))
#		oflist.append(int(mess[i]['created']))
#		oflist.append(int(mess[i]['deleted']))
#		oflist.append(int(mess[i]['id']))
#		oflist.append(bool(mess[i]['isTradeCancled']))
#		oflist.append(int(mess[i]['lotsRemaining']))
#		oflist.append(str(mess[i]['offer']))
#		oflist.append(int(mess[i]['offerAcceptedID']))
#		oflist.append(int(mess[i]['receiverID']))
#		oflist.append(int(mess[i]['remainingTime']))
#		oflist.append(int(mess[i]['removed']))
#		oflist.append(int(mess[i]['senderID']))
#		oflist.append(mess[i]['senderName'].__str__())
#		oflist.append(int(mess[i]['slotPos']))
#		oflist.append(int(mess[i]['slotType']))
#		oflist.append(int(mess[i]['type']))		
#		print 'krok' + i.__str__() + str(mess[i]['offer'])

        #coolDownTime    = int(mess[i]['coolDownTime'])
        #created         = int(mess[i]['created'])
        #deleted         = int(mess[i]['deleted'])
        #id              = int(mess[i]['id'])
        #isTradeCancled  = bool(mess[i]['isTradeCancled'])
        #lotsRemaining   = int(mess[i]['lotsRemaining'])
        #offer           = str(mess[i]['offer'])
        #offerAcceptedID = int(mess[i]['offerAcceptedID'])
        #receiverID      = int(mess[i]['receiverID'])
        #remainingTime   = int(mess[i]['remainingTime'])
        #removed         = int(mess[i]['removed'])
        #senderID        = int(mess[i]['senderID'])
        #senderName      = mess[i]['senderName']
        #slotPos         = int(mess[i]['slotPos'])
        #slotType        = int(mess[i]['slotType'])
        #type            = int(mess[i]['type'])



	#	dbc.execute(q, (int(mess[i]['coolDownTime']), int(mess[i]['created']), int(mess[i]['deleted']), int(mess[i]['id']), bool(mess[i]['isTradeCancled']), int(mess[i]['lotsRemaining']), str(mess[i]['offer']), int(mess[i]['offerAcceptedID']), int(mess[i]['receiverID']), int(mess[i]['remainingTime']), int(mess[i]['removed']), int(mess[i]['senderID']), mess[i]['senderName'].__str__(), int(mess[i]['slotPos']), int(mess[i]['slotType']), int(mess[i]['type'])))
	#db.commit()	
#	dbc.executemany(q, oflist)
#	db.commit()

def error(ctx, flow):
    """
        Called when a flow error has occured, e.g. invalid server responses, or
        interrupted connections. This is distinct from a valid server HTTP error
        response, which is simply a response with an HTTP error code.
    """
    ctx.log("error")
    print("error")

def clientdisconnect(ctx, client_disconnect):
    """
        Called when a client disconnects from the proxy.
    """
    ctx.log("clientdisconnect")
    print("clientdisconnect")

def done(ctx):
    """
        Called once on script shutdown, after any other events.
    """
    ctx.log("done")
    print("done")
```python

0

wszystko jasne: po prostu przychodziły kolejne dane i do uzyskania pożądanych rezultatów trzeba było odczepiać osobny thread, żeby miał odpowiednio dużo czasu na przetrawienie danych .

0

zamiast startować nowy wątek parsujący dane, wystarczy użyć dekoratora @concurrent

0

heh, teraz już tak. w momencie gdy to pisałem dekorator jeszcze nie istniał :)

1 użytkowników online, w tym zalogowanych: 0, gości: 1