File manager

U

e5d�$�@sXdd
lZdd
l
Zdd
lZdg
Ze�dd�ZGdd�d�ZGdd�d�ZGdd	�d	�Z	d
S)
�N�RobotFileParser�RequestRatezrequests secondsc@sneZ
dZddd�
Zdd�Zdd�Zdd	�Zd
d�Zdd
�Zdd�Z	dd�Z
dd�Zdd�Zdd�Z
dd�ZdS)r�cCs2g|_g|_
d|_d
|_d
|_|�|
�

d|_dS)NFr
)�entries�sitemaps�
default_entry�disallow_all�	allow_all�set_url�last_checked��self�url�r�*/usr/lib64/python3.8/urllib/robotparser.py�__init__s







zRobotFileParser.__init__c


Cs|jS�
N)
r�
r
rrr�mtime%szRobotFileParser.mtimec
Csd
dl}
|
��|_
dS)Nr
)�timer)r
rrrr�modified.s
zRobotFileParser.modifiedcCs&|
|_t
j�|
�
d
d�\|_|_dS)N�
�)r�urllib�parse�urlparseZhost�pathrrrrr
6s
zRobotFileParser.set_urlc

Cs�ztj
�|j�
}
WnRtjjk
rd
}
z0|jd
kr:d|_n|jdkrT|jdkrTd|_W5d}~XYnX|
�	�}|�
|�d�
���

dS)N)i�
i�
Ti�
i�
zutf-8)
rZrequestZurlopenr�errorZ	HTTPError�coderr	�readr�decode�
splitlines)r
�
f�err�rawrrrr;s







zRobotFileParser.readcCs,d
|
jkr|j
dkr(|
|_
n|j�|
�

dS�N�
*)�
useragentsrr�append)r
�entryrrr�
_add_entryHs


zRobotFileParser._add_entrycCsPd
}t�}|�
�
|
D�]}|sP|dkr4t�}d
}n|dkrP|�|�

t�}d
}|�d�
}|d
krn|d|�}|��}|s|q|�dd�}t|�
dkr|d
����|d
<tj	�
|d���
|d<|d
dkr�|dkr�|�|�

t�}|j�|d�

d}q|d
dk�
r.|d
k�r6|j
�t|dd��

d}q|d
d	k�
rb|d
k�r6|j
�t|dd
��

d}q|d
dk�
r�|d
k�r6|d�����
r�t|d�
|_d}q|d
dk�r|d
k�r6|d�d
�
}t|�
dk�r|d
�����r|d�����rtt|d
�
t|d�
�|_d}q|d
dkr|j�|d�

q|dk�rL|�|�

dS)Nr
r��
#�
:z
user-agentZdisallowFZallowTzcrawl-delayzrequest-rate�
/Zsitemap)�Entryrr*�find�strip�split�len�lowerrr�unquoter'r(�	rulelines�RuleLine�isdigit�int�delayr�req_rater)r
�lines�stater)�line�
iZnumbersrrrrQsj
















































 
�




zRobotFileParser.parsecCs�|jr
d
S|j
rdS|jsd
Stj�tj�|�
�
}tj�dd|j|j	|j
|jf�
}tj�|�
}|sfd}|j
D]}|�|
�
rl|�|�

Sql|jr�|j�|�
SdS)NFTrr.)rr	rrrrr5�
urlunparserZparamsZqueryZfragment�quoter�
applies_to�	allowancer)r
�	useragentrZ
parsed_urlr)rrr�	can_fetch�s*





�







zRobotFileParser.can_fetchcCs>|��sdS|j
D]}|�|
�
r|j
Sq|jr:|jjSdSr)rrrBr:r�r
rDr)rrr�crawl_delay�s









zRobotFileParser.crawl_delaycCs>|��sdS|j
D]}|�|
�
r|j
Sq|jr:|jjSdSr)rrrBr;rrFrrr�request_rate�s









zRobotFileParser.request_ratec


Cs|js
dS|jSr)
rrrrr�	site_maps�s


zRobotFileParser.site_mapsc
Cs,|j}
|j
dk	r|
|j
g
}
d
�tt|
��
S)Nz

)rr�join�map�str)r
rrrr�__str__�s




zRobotFileParser.__str__N)
r)�__name__�
__module__�__qualname__rrrr
rr*rrErGrHrIrMrrrrrs
		
	I

c@s$eZ
dZd
d�Zdd�Zdd�ZdS)r7cCs<|
d
kr|sd}tj
�tj
�|
�
�
}
tj
�|
�
|_||_dS)NrT)rrr@rrArrC)r
rrCrrrr�s




zRuleLine.__init__cCs|jd
kp|
�
|j�
Sr%)r�
startswith)r
�filenamerrrrB�s
zRuleLine.applies_toc

Cs|jr
d
ndd|j
S)NZAllowZDisallowz: )rCrrrrrrM�s
zRuleLine.__str__N)rNrOrPrrBrMrrrrr7�sr7c@s,eZ
dZd
d�Zdd�Zdd�Zdd�Zd	S)
r/c

Csg|_g|_
d|_d|_dSr)r'r6r:r;rrrrr�s



zEntry.__init__c
Cs�g}
|jD]}|
�
d
|���

q
|jdk	r<|
�
d|j���

|jdk	rf|j}|
�
d|j�d|j���

|
�tt|j	��

d�
|
�
S)NzUser-agent: z
Crawl-delay: zRequest-rate: r.�

)r'r(r:r;ZrequestsZseconds�extendrKrLr6rJ)r
Zret�agentZraterrrrM�s












z
Entry.__str__cCsF|
�d
�
d�
�}
|jD](}|dkr*
dS|�
�}||
kr
dSqdS)Nr.r
r&TF)r2r4r')r
rDrUrrrrB�s






zEntry.applies_tocCs$|jD]}|�
|
�
r|j
Sqd
S)NT)r6rBrC)r
rRr>rrrrC

s




zEntry.allowanceN)rNrOrPrrMrBrCrrrrr/�s
r/)
�collectionsZurllib.parserZurllib.request�__all__�
namedtuplerrr7r/rrrr�<module>
s

B