← Back
Editing: robotparser.cpython-36.opt-2.pyc
3 \�" � @ sX d dl Z d dlZd dlZdgZe jdd�ZG dd� d�ZG dd� d�ZG dd � d �Z dS ) � N�RobotFileParser� RequestRatezrequests secondsc @ sf e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dd� Z dd� Z dd� Z dd� Z dS )r � c C s, g | _ d | _d| _d| _| j|� d| _d S )NFr )�entries� default_entry� disallow_all� allow_all�set_url� last_checked)�self�url� r �*/usr/lib64/python3.6/urllib/robotparser.py�__init__ s zRobotFileParser.__init__c C s | j S )N)r )r r r r �mtime$ s zRobotFileParser.mtimec C s dd l }|j � | _d S )Nr )�timer )r r r r r �modified- s zRobotFileParser.modifiedc C s& || _ tjj|�dd� \| _| _d S )N� � )r �urllib�parse�urlparse�host�path)r r r r r r 5 s zRobotFileParser.set_urlc C s� yt jj| j�}W nR t jjk rd } z2|jdkr:d| _n|jdkrT|jdk rTd| _W Y d d }~X n X |j � }| j |j d�j � � d S )N� � Ti� i� zutf-8)r r ) r ZrequestZurlopenr �errorZ HTTPError�coder r �readr �decode� splitlines)r �f�err�rawr r r r : s zRobotFileParser.readc C s, d|j kr | jd kr(|| _n | jj|� d S )N�*)� useragentsr r �append)r �entryr r r � _add_entryG s zRobotFileParser._add_entryc C s6 d}t � }| j� �x|D �]�}|sT|dkr8t � }d}n |dkrT| j|� t � }d}|jd�}|dkrr|d |� }|j� }|s�q|jdd�}t|�dkr|d j� j� |d< tj j |d j� �|d< |d dk�r |dkr�| j|� t � }|j j |d � d}q|d dk�r4|dk�r|j j t|d d�� d}q|d d k�rh|dk�r|j j t|d d �� d}q|d d k�r�|dk�r|d j� j� �r�t|d �|_d}q|d d kr|dkr|d jd �}t|�dk�r|d j� j� �r|d j� j� �rtt|d �t|d ��|_d}qW |dk�r2| j|� d S )Nr r � �#�:z user-agentZdisallowFZallowTz crawl-delayz request-rate�/)�Entryr r( �find�strip�split�len�lowerr r �unquoter% r& � rulelines�RuleLine�isdigit�int�delayr �req_rate)r �lines�stater' �line�iZnumbersr r r r P sd zRobotFileParser.parsec C s� | j r dS | jrdS | js dS tjjtjj|��}tjjdd|j|j |j |j f�}tjj |�}|sfd}x"| j D ]}|j|�rn|j|�S qnW | jr�| jj|�S dS )NFTr r, )r r r r r r r3 � urlunparser ZparamsZqueryZfragment�quoter � applies_to� allowancer )r � useragentr Z parsed_urlr' r r r � can_fetch� s$ zRobotFileParser.can_fetchc C s4 | j � s d S x | jD ]}|j|�r|jS qW | jjS )N)r r r@ r8 r )r rB r' r r r � crawl_delay� s zRobotFileParser.crawl_delayc C s4 | j � s d S x | jD ]}|j|�r|jS qW | jjS )N)r r r@ r9 r )r rB r' r r r � request_rate� s z RobotFileParser.request_ratec C s0 | j }| jd k r || jg }djtt|��d S )N� )r r �join�map�str)r r r r r �__str__� s zRobotFileParser.__str__N)r )�__name__� __module__� __qualname__r r r r r r( r rC rD rE rJ r r r r r s C c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )r5 c C s> |dkr| rd}t jjt jj|��}t jj|�| _|| _d S )Nr T)r r r> r r? r rA )r r rA r r r r � s zRuleLine.__init__c C s | j dkp|j| j �S )Nr$ )r � startswith)r �filenamer r r r@ � s zRuleLine.applies_toc C s | j r dndd | j S )NZAllowZDisallowz: )rA r )r r r r rJ � s zRuleLine.__str__N)rK rL rM r r@ rJ r r r r r5 � s r5 c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r- c C s g | _ g | _d | _d | _d S )N)r% r4 r8 r9 )r r r r r � s zEntry.__init__c C s� g }x | j D ]}|jd|� �� q W | jd k r@|jd| j� �� | jd k rj| j}|jd|j� d|j� �� |jtt| j �� |jd� dj |�S )Nz User-agent: z Crawl-delay: zRequest-rate: r, r rF ) r% r& r8 r9 ZrequestsZseconds�extendrH rI r4 rG )r Zret�agentZrater r r rJ � s z Entry.__str__c C sF |j d�d j� }x.| jD ]$}|dkr*dS |j� }||krdS qW dS )Nr, r r$ TF)r0 r2 r% )r rB rQ r r r r@ � s zEntry.applies_toc C s$ x | j D ]}|j|�r|jS qW dS )NT)r4 r@ rA )r rO r< r r r rA � s zEntry.allowanceN)rK rL rM r rJ r@ rA r r r r r- � s r- ) � collectionsZ urllib.parser Zurllib.request�__all__� namedtupler r r5 r- r r r r �<module> s 2
Save File
Cancel