^OPc@sddlZddlZddlZddlZddlZddljZddljZddl j Z ddl j Z ddl jZddljZddlZddljjZddlmZmZmZdZdZdZdZdZdZ dZ!d Z"d e#fd YZ$dS(iN(tEmptyIt PKG_DIR_MODEtPKG_FILE_BUFSIZiiiissort.iicCsytj|tWntk r}|j|kr`|jtjks\tjj|jr`dS|jtj tj fkrt j |jq|jtjks|j|krqnXdS(s~Create a directory at the specified location if it does not already exist (including any parent directories). N( tostmakedirsRtEnvironmentErrortfilenameterrnotEEXISTtpathtexiststEACCEStEROFSt search_errorst$ProblematicPermissionsIndexException(tpathnamete((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyR;s!!tIndexercBs1eZdZdZdededZedZ edZ edZ dZ dZ dZd Zd Zd Zed Zd ZdZdZdddZddZddZdZddZdZedZddedZedZdZ RS(sIndexer is a class designed to index a set of manifests or pkg plans and provide a compact representation on disk, which is quickly searchable.s VERSION: cCs|d|_d|_d|_||_||_||_||_||_|jdkrotj t dnit j t j d6t j t jd6t jt jd|jd|jd6t j t jd6t jt jd 6t jt jd 6|_|jd|_|jd|_|jd|_|jd|_|jd |_|jd |_t jt j |j|jd <|jd |_!||_"t#j$j%|j"d |_&t'j(t#j$j%|j"d dt'j)dt'j*dtj+|_,d|_-t.|_/t0|_1d|_3|dkr0t4j5|_6n ||_6t7|_8d|_9d|_:d|_;i|_<i|_=d|_>dS(Nis)sort_file_max_size must be greater than 0tfast_addt fast_removetbuild_functiontdecode_functiontmanft full_fmrit main_dictttoken_byte_offsett fmri_offsetstTMPtlockt set_lockstrt get_lockstrt failure_exc(?t _num_keyst_num_manifestst _num_entriestget_manifest_functget_manifest_path_functexcludest _Indexer__logtsort_file_max_sizeR tIndexingExceptiont_tsst IndexStoreSettFAST_ADDt FAST_REMOVEtIndexStoreListDictt MANIFEST_LISTt_Indexer__build_fmrit_Indexer__decode_fmritFULL_FMRI_FILEtIndexStoreMainDictt MAIN_FILEtIndexStoreDictMutabletBYTE_OFFSET_FILEt _data_dictt_data_fast_addt_data_fast_removet _data_manft_data_full_fmrit_data_main_dictt_data_token_offsett InvertedDicttFMRI_OFFSETS_FILEt_data_fmri_offsetst _index_dirRR tjoint_tmp_dirtlockfiletLockFiletgeneric_lock_set_strtgeneric_lock_get_strtIndexLockedExceptiont_Indexer__lockfilet_indexed_manifeststTruet server_repotFalset empty_indextNonetfile_version_numbertprogresstNullProgressTrackert _progtracktFILE_OPEN_TIMEOUT_SECSt_file_timeout_secst_sort_fht_sort_file_numt_sort_file_bytestat_fhtst_fht old_out_token(tselft index_dirR#R$t progtrackR%tlogR'((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt__init__Tsb                         cCs|jdtdtS(sZTurn fmris into strings correctly while writing out the fmri offsets file.tanarchytinclude_scheme(tget_fmriRKRM(tpfmri((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt __decode_fmriscCs tj|S(s6Build fmris while reading the fmri offset information.(tfmritPkgFmri(ts((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt __build_fmriscCstjjtj|dS(s5 Private method for building versions from a string. N(tpkgtversiontVersionturllibtunquoteRO(tvers((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt_build_versionscCsDtj|jj||j}|j}|dkrLt|_t |_ dS||_z|j |j ylxe|jjD]T}||j ks||jkr|j|j q{n|j|j|j q{WWn|jdjnXWdx6|jjD]%}||j kr!qn|jqW|j|j XdS(s Opens all index files using consistent_open and reads all of them into memory except the main dictionary file to avoid inefficient memory usage.RN(R*tconsistent_openR7tvaluesRURSROtINITIAL_VERSION_NUMBERRPRKRNt job_starttJOB_READ_SEARCHR<R=tjob_add_progresstread_dict_filetclose_file_handletjob_done(R\t directorytrestpttd((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt_read_input_indexess6         cCs|jjd|_tjj|jtt|j d}t |ddt }g|D]}t j j||f^q[}|j|jt |ddt }|jd|D|jdS(ssUtility fuction used to close and sort the temporary files used to produce a sorted main_dict file.iitrbt bufferingtwbcss|]\}}|VqdS(N((t.0ttoktline((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pys sN(RVtcloseRXRR RBRCtSORT_FILE_PREFIXtstrRWtfileRR*R3tparse_main_dict_line_for_tokentsortt writelines(R\t tmp_file_namettmp_fhRtl((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt__close_sort_fhs  (  c Cs"|jj|}|}x|jD]}|\}}}}||||t||fgfgfgfg} tjj|| } t| |j|j kr|j t t j j|jtt|jddt|_|jd7_n|jj| |jt| 7_q%WdS(s/Adds tokens, and the actions generating them, to the current temporary sort file. The "pfmri" parameter is the fmri the information is coming from. The "new_dict" parameter maps tokens to the information about the action.RRiN(R:tget_id_and_addtkeystlistR*R3ttransform_main_dict_linetlenRXR't_Indexer__close_sort_fhtopenRR RBRCRRRWRRVtwrite( R\Rdtnew_dicttp_idttok_tupRt action_typetsubtypetfvtlstRh((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt _add_termss&  (     c Csvt|jj}t|jj}|\}}x|D]}|\}}|r|jj|jdt|jdtdt} |jj | r|d8}q|d7}n|r7|jj |jdt|jdtdt} |jj | r|d8}q|d7}q7q7W|t kr)tS|j j |j jdt|x|D]}|\}}|r|jdtdt} |jj |  st|jj | r|jj | q|jj| n|rB|jdtdt} |jj |  s t|jj | r/|jj | qB|jj| n|j j|j jqRW|j j|j jtS(sUpdates the log of packages which have been installed or removed since the last time the index has been rebuilt. There are two axes to consider: whether the package is being added or removed; whether this version of the package is already present in an update log. Case 1: The package is being installed and is not in the update log. In this case, the new package is simply added to the install update log. Case 2: The package is being installed and is in the removal update log. In this case, the package is removed from the remove update log. This has the effect of exposing the entries in the existing index to the user. Case 3: The package is being removed and is not in the update log. In this case, the new package is simply added to the removed update log. Case 4: The package is being removed and is in the installed update log. In this case, the package is removed from the install update log. The "filters_pkgplan_list" parameter is a tuple of a list of filters, which are currently ignored, and a list of pkgplans that indicated which versions of a package are being added or removed.RaRbitgoal(RR8t_setR9R;t add_entityRcRKRMt has_entityt remove_entitytMAX_FAST_INDEXED_PKGSRSRttJOB_UPDATE_SEARCHtAssertionErrorRvRy( R\tfilters_pkgplan_listt nfast_addt nfast_removetfilterst pkgplan_listtptd_fmrito_fmritd_tmpto_tmp((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt _fast_updates^                    cCsg}xz|D]r}|jj|jdttjj|j||jd|j }|j |||j j |j j q W|S(scTakes a list of fmris and updates the internal storage to reflect the new packages.RaR_(R;RRcRKtmanifesttManifestt search_dictR$R%R&RRSRvtJOB_REBUILD_SEARCH(R\tfmrist removed_pathst added_fmriR((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyt_process_fmrisns     cCs|jdk r:|j|kr:td||jfn||_|j}t|}|jj||x:|D]2\}}|jj|jj dd||j krt t j j|d|d|j |t t j j|d| d|j| s  cCst|_t|_|j|jtj|j|jdy%t j |jdt |jWn:t k r}|j t jkrtj|jqnX|j|t|t|_dS(sRemoves any existing index directory and rebuilds the index based on the fmris and manifests provided as an argument. The "tmp_index_dir" parameter allows for a different directory than the default to be used.s.oldN(RsRPRKRNRRtportabletrenameRARRRtOSErrorRR R RRR RM(R\RRR((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyR Ts     cCst}t}t|jxw|jjD]f}tjj|j|j}tjj |rht }nt }|r)|r)t j |jq)q)W|rdS|j rtdnt|_ x-|jjD]}|j|j|j qWdS(sSeeds the index directory with empty stubs if the directory is consistently empty. Does not overwrite existing indexes.Ns1Got file_version_number other than None in setup.(RMRRAR7RrRR RBRR RKR RRPRRsR(R\tabsenttpresentR}t file_path((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pytsetupus(      cCstd|jD}tjd}y|j|WnAtk r}tjjtjj ||j sy|SnXz|j |Wd|j X|S(sCheck to see whether the catalog has fmris which have not been indexed. 'index_root' is the path to the index to check against. 'cat' is the catalog to check for new fmris.css|]}|jVqdS(N(tremove_publisher(Rtf((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pys stfull_fmri_listN( tsetRR*R+RtIOErrorRR R RBRt'read_and_discard_matching_from_argumentRx(t index_roottcattfmri_settdataR((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pytcheck_for_updatess  cCs|s|j}n|s$|j}n||k s7tx|jjD]y}|r||jksG||jksG||jkrqGqGtj t j j ||j t j j ||j qGW|sy tjt j j |dWn$tk rntk rnXxS|jjD]B\}}tj t j j |d|t j j |d|q!WxV|jjD]B\}}tj t j j |d|t j j |d|qwWntj|dS(s>Moves the indexes from a temporary directory to the permanent one. The "source_dir" parameter is the directory containing the new information. The "dest_dir" parameter is the directory containing the old information. The "fast_update" parameter determines whether the main dictionary and the token byte offset files are moved. This is used so that when only the update logs are touched, the large files don't need to be moved.RjRRN(RCRARR7RrR<R=R@RtmoveRR RBRRRRRYtitemsRZ(R\Rtdest_dirRR}RRR((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyR s:       !cCsy|jjd|Wntk r}|jtjkrstjj|jsmt |j|jd|Sn|jtj tj fkrt j |jnnXdS(s`Locks the index in preparation for an index-modifying operation. Raises an IndexLockedException exception on failure. 'blocking' is an optional boolean value indicating whether to block until the lock can be obtained or to raise an exception immediately if it cannot be.tblockingN(RIRRRtENOENTRR R RARR R R RR(R\R.R((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyRs  cCs|jjdS(sUnlocks the index.N(RIR(R\((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyRsN(!t__name__t __module__t__doc__tfile_version_stringRORtSORT_FILE_MAX_SIZER`t staticmethodR1R0RpR~RRRRRRRRRRRRRR RR*RMR RR(((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyRMs6  Q #   ^  0 ^ i  ! 3 ii(%RRtplatformRRmtpkg.fmriRft pkg.lockfileRDt pkg.manifestRt pkg.portableRtpkg.search_storagetsearch_storageR*tpkg.search_errorsR t pkg.versionRjtpkg.client.progresstclientRQtpkg.miscRRRRR RsRTRRR4RtobjectR(((s1/usr/lib/python2.7/vendor-packages/pkg/indexer.pyts,