← Back
Editing: robotparser.cpython-37.opt-2.pyc
B YI~d�" � @ sX d dl Z d dlZd dlZdgZe �dd�ZG dd� d�ZG dd� d�ZG dd � d �Z dS ) � N�RobotFileParser� RequestRatezrequests secondsc @ sf e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dd� Z dd� Z dd� Z dd� Z dS )r � c C s, g | _ d | _d| _d| _| �|� d| _d S )NFr )�entries� default_entry� disallow_all� allow_all�set_url� last_checked)�self�url� r �7/opt/alt/python37/lib64/python3.7/urllib/robotparser.py�__init__ s zRobotFileParser.__init__c C s | j S )N)r )r r r r �mtime$ s zRobotFileParser.mtimec C s dd l }|� � | _d S )Nr )�timer )r r r r r �modified- s zRobotFileParser.modifiedc C s& || _ tj�|�dd� \| _| _d S )N� � )r �urllib�parse�urlparseZhost�path)r r r r r r 5 s zRobotFileParser.set_urlc C s� yt j�| j�}W nR t jjk rd } z0|jdkr:d| _n|jdkrT|jdk rTd| _W d d }~X Y n X |� � }| � |� d�� � � d S )N)i� i� Ti� i� zutf-8) r ZrequestZurlopenr �errorZ HTTPError�coder r �readr �decode� splitlines)r �f�err�rawr r r r : s zRobotFileParser.readc C s, d|j kr | jd kr(|| _n | j�|� d S )N�*)� useragentsr r �append)r �entryr r r � _add_entryG s zRobotFileParser._add_entryc C s6 d}t � }| �� �x|D �]�}|sT|dkr8t � }d}n |dkrT| �|� t � }d}|�d�}|dkrr|d |� }|�� }|s�q|�dd�}t|�dkr|d �� �� |d< tj � |d �� �|d< |d dk�r |dkr�| �|� t � }|j � |d � d}q|d dk�r4|dk�r|j � t|d d�� d}q|d d k�rh|dk�r|j � t|d d �� d}q|d d k�r�|dk�r|d �� �� �r�t|d �|_d}q|d d kr|dkr|d �d �}t|�dk�r|d �� �� �r|d �� �� �rtt|d �t|d ��|_d}qW |dk�r2| �|� d S )Nr r � �#�:z user-agentZdisallowFZallowTz crawl-delayz request-rate�/)�Entryr r% �find�strip�split�len�lowerr r �unquoter" r# � rulelines�RuleLine�isdigit�int�delayr �req_rate)r �lines�stater$ �line�iZnumbersr r r r P sd zRobotFileParser.parsec C s� | j r dS | jrdS | js dS tj�tj�|��}tj�dd|j|j |j |j f�}tj� |�}|sfd}x"| j D ]}|�|�rn|�|�S qnW | jr�| j�|�S dS )NFTr r) )r r r r r r r0 � urlunparser �paramsZqueryZfragment�quoter � applies_to� allowancer )r � useragentr Z parsed_urlr$ r r r � can_fetch� s$ zRobotFileParser.can_fetchc C s> | � � s d S x | jD ]}|�|�r|jS qW | jr:| jjS d S )N)r r r> r5 r )r r@ r$ r r r � crawl_delay� s zRobotFileParser.crawl_delayc C s> | � � s d S x | jD ]}|�|�r|jS qW | jr:| jjS d S )N)r r r> r6 r )r r@ r$ r r r � request_rate� s z RobotFileParser.request_ratec C s0 | j }| jd k r || jg }d�tt|��d S )N� )r r �join�map�str)r r r r r �__str__� s zRobotFileParser.__str__N)r )�__name__� __module__� __qualname__r r r r r r% r rA rB rC rH r r r r r s C c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )r2 c C s< |dkr|sd}t j�t j�|��}t j�|�| _|| _d S )Nr T)r r r; r r= r r? )r r r? r r r r � s zRuleLine.__init__c C s | j dkp|�| j �S )Nr! )r � startswith)r �filenamer r r r> � s zRuleLine.applies_toc C s | j r dndd | j S )NZAllowZDisallowz: )r? r )r r r r rH � s zRuleLine.__str__N)rI rJ rK r r> rH r r r r r2 � s r2 c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r* c C s g | _ g | _d | _d | _d S )N)r" r1 r5 r6 )r r r r r � s zEntry.__init__c C s� g }x | j D ]}|�d|� �� q W | jd k r@|�d| j� �� | jd k rj| j}|�d|j� d|j� �� |�tt| j �� |�d� d� |�S )Nz User-agent: z Crawl-delay: zRequest-rate: r) r rD ) r" r# r5 r6 ZrequestsZseconds�extendrF rG r1 rE )r �ret�agentZrater r r rH � s z Entry.__str__c C sF |� d�d �� }x.| jD ]$}|dkr*dS |�� }||krdS qW dS )Nr) r r! TF)r- r/ r" )r r@ rP r r r r> � s zEntry.applies_toc C s$ x | j D ]}|�|�r|jS qW dS )NT)r1 r> r? )r rM r9 r r r r? � s zEntry.allowanceN)rI rJ rK r rH r> r? r r r r r* � s r* ) � collectionsZ urllib.parser Zurllib.request�__all__� namedtupler r r2 r* r r r r �<module> s 6
Save File
Cancel