Python csv.field_size_limit方法代码示例

本文整理汇总了Python中csv.field_size_limit方法的典型用法代码示例。如果您正苦于以下问题:Python csv.field_size_limit方法的具体用法?Python csv.field_size_limit怎么用?Python csv.field_size_limit使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在模块csv的用法示例。

在下文中一共展示了csv.field_size_limit方法的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: Records

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def Records(self):
    """Reads the CSV data file and generates row records.
    Yields:
      Lists of strings
    Raises:
      ResumeError: If the progress database and data file indicate a different
        number of rows.
    """
    csv_file = self.openfile(self.csv_filename, 'rb')
    reader = self.create_csv_reader(csv_file, skipinitialspace=True)
    try:
      for record in reader:
        yield record
    except csv.Error, e:
      if e.args and e.args[0].startswith('field larger than field limit'):
        raise FieldSizeLimitError(csv.field_size_limit())
      else:
        raise 
开发者ID:elsigh,项目名称:browserscope,代码行数:23,代码来源:bulkloader.py


示例2: test_read_bigfield

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def test_read_bigfield(self):
        # This exercises the buffer realloc functionality and field size
        # limits.
        limit = csv.field_size_limit()
        try:
            size = 50000
            bigstring = 'X' * size
            bigline = '%s,%s' % (bigstring, bigstring)
            self._read_test([bigline], [[bigstring, bigstring]])
            csv.field_size_limit(size)
            self._read_test([bigline], [[bigstring, bigstring]])
            self.assertEqual(csv.field_size_limit(), size)
            csv.field_size_limit(size-1)
            self.assertRaises(csv.Error, self._read_test, [bigline], [])
            self.assertRaises(TypeError, csv.field_size_limit, None)
            self.assertRaises(TypeError, csv.field_size_limit, 1, None)
        finally:
            csv.field_size_limit(limit) 
开发者ID:IronLanguages,项目名称:ironpython2,代码行数:20,代码来源:test_csv.py


示例3: index

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def index(sqlite_filename, tsv_filename):
    logger.info('Reading tsv file %s', tsv_filename)
    # https://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072
    # https://github.com/balta2ar/brotab/issues/25
    # It should work on Python 3 and Python 2, on any CPU / OS.
    csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
    with open(tsv_filename, encoding='utf-8') as tsv_file:
        lines = [tuple(line) for line in csv.reader(tsv_file, delimiter='\t',
                                                    quoting=csv.QUOTE_NONE)]
    logger.info(
        'Creating sqlite DB filename %s from tsv %s (%s lines)',
        sqlite_filename, tsv_filename, len(lines))
    conn = sqlite3.connect(sqlite_filename)
    cursor = conn.cursor()
    with suppress(sqlite3.OperationalError):
        cursor.execute('drop table tabs;')
    cursor.execute(
        'create virtual table tabs using fts5('
        '    tab_id, title, url, body, tokenize="porter unicode61");')
    cursor.executemany('insert into tabs values (?, ?, ?, ?)', lines)
    conn.commit()
    conn.close() 
开发者ID:balta2ar,项目名称:brotab,代码行数:26,代码来源:index.py


示例4: load_lookup_file_from_disk

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def load_lookup_file_from_disk(file_path):
    """
    parse the lookup file from the given path and return the result
    Args:
        file_path (string): the path to the lookup file
    Returns:
        lookup_data (dict): result from the csv parser
    """
    if not file_exists(file_path):
        raise RuntimeError('Not valid filepath: {}'.format(file_path))
    try:
        with open(file_path, mode='r') as f:
            reader = csv.DictReader(f)
            csv.field_size_limit(CSV_FILESIZE_LIMIT)
            lookup_data = reader.next()
    except Exception as e:
        raise RuntimeError('Error reading model file: %s, %s' % (file_path, str(e)))
    return lookup_data 
开发者ID:nccgroup,项目名称:Splunking-Crime,代码行数:24,代码来源:lookups_util.py


示例5: _load_sentence_file

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def _load_sentence_file(self, filename):
        csv.field_size_limit(sys.maxsize)
        opener = gzip.open if filename.endswith('.gz') else open
        entities = dict()
        with opener(filename) as csvfile:
                reader = csv.reader(csvfile, delimiter='\t',quoting=csv.QUOTE_NONE)
                for row in reader:
                        qid = row[0].strip()
                        sentence_json = row[1].strip()
                        if sentence_json:
                                payload = json.loads(sentence_json)
                                annotations = payload['annotations']
                                sentence_entities = [ x['id'] for x in annotations]
                                sentence_entities = [ str(x) for x in sentence_entities]
                                entities[qid] = sentence_entities
                        else:
                                entities[qid] = []
        return entities 
开发者ID:rmit-ir,项目名称:SummaryRank,代码行数:20,代码来源:resources.py


示例6: init

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def init(args):
    """
        Load data, build model, create optimizer, create vars to hold metrics, etc.
    """
    #need to handle really large text fields
    csv.field_size_limit(sys.maxsize)
    #load vocab and other lookups
    desc_embed = args.lmbda > 0
    print("loading lookups...")
    dicts = datasets.load_lookups(args, desc_embed=desc_embed)
    model = tools.pick_model(args, dicts)
    print(model)
    if not args.test_model:
        optimizer = optim.Adam(model.parameters(), weight_decay=args.weight_decay, lr=args.lr)
    else:
        optimizer = None
    params = tools.make_param_dict(args)
    return args, model, optimizer, params, dicts 
开发者ID:jamesmullenbach,项目名称:caml-mimic,代码行数:25,代码来源:training.py


示例7: read_file

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def read_file(data_dir, with_evaluation):
    data = []
    target = []
    with open(join(data_dir, 'dataset.csv'), 'rt', encoding='utf-8') as csvfile:
        csv.field_size_limit(500 * 1024 * 1024)
        reader = csv.reader(csvfile)
        for row in reader:
            if data_dir == './agnews':
                doc = row[1] + '. ' + row[2]
                data.append(doc)
                target.append(int(row[0]) - 1)
            elif data_dir == './yelp':
                data.append(row[1])
                target.append(int(row[0]) - 1)
    if with_evaluation:
        y = np.asarray(target)
        assert len(data) == len(y)
        assert set(range(len(np.unique(y)))) == set(np.unique(y))
    else:
        y = None
    return data, y 
开发者ID:yumeng5,项目名称:WeSTClass,代码行数:23,代码来源:load_data.py


示例8: _increase_csv_field_max_size

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def _increase_csv_field_max_size():
    """Makes document entry in dataset as big as possible
    References
    ----------
    https://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072
    """
    max_int = sys.maxsize
    while True:
        try:
            csv.field_size_limit(max_int)
            break
        except OverflowError:
            max_int = int(max_int / 10) 
开发者ID:machine-intelligence-laboratory,项目名称:TopicNet,代码行数:20,代码来源:dataset.py


示例9: unicode_csv_reader

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def unicode_csv_reader(unicode_csv_data: TextIOWrapper, **kwargs: Any) -> Any:
    r"""Since the standard csv library does not handle unicode in Python 2, we need a wrapper.
    Borrowed and slightly modified from the Python docs:
    https://docs.python.org/2/library/csv.html#csv-examples
    Args:
        unicode_csv_data (TextIOWrapper): unicode csv data (see example below)
    Examples:
        >>> from torchaudio.datasets.utils import unicode_csv_reader
        >>> import io
        >>> with io.open(data_path, encoding="utf8") as f:
        >>>     reader = unicode_csv_reader(f)
    """
    # Fix field larger than field limit error
    maxInt = sys.maxsize
    while True:
        # decrease the maxInt value by factor 10
        # as long as the OverflowError occurs.
        try:
            csv.field_size_limit(maxInt)
            break
        except OverflowError:
            maxInt = int(maxInt / 10)
    csv.field_size_limit(maxInt)
    for line in csv.reader(unicode_csv_data, **kwargs):
        yield line 
开发者ID:pytorch,项目名称:audio,代码行数:30,代码来源:utils.py


示例10: __init__

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def __init__(self, sf):
        # Set csv max reading size to the platform's max size available.
        csv.field_size_limit(sys.maxsize)
        self.sf = sf 
开发者ID:singer-io,项目名称:tap-salesforce,代码行数:6,代码来源:bulk.py


示例11: ContentGenerator

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def ContentGenerator(csv_file,
                     batch_size,
                     create_csv_reader=csv.reader,
                     create_csv_writer=csv.writer):
  """Retrieves CSV data up to a batch size at a time.
  Args:
    csv_file: A file-like object for reading CSV data.
    batch_size: Maximum number of CSV rows to yield on each iteration.
    create_csv_reader, create_csv_writer: Used for dependency injection.
  Yields:
    Tuple (entity_count, csv_content) where:
      entity_count: Number of entities contained in the csv_content. Will be
        less than or equal to the batch_size and greater than 0.
      csv_content: String containing the CSV content containing the next
        entity_count entities.
  """
  try:
    csv.field_size_limit(800000)
  except AttributeError:
    pass
  reader = create_csv_reader(csv_file, skipinitialspace=True)
  exhausted = False
  while not exhausted:
    rows_written = 0
    content = StringIO.StringIO()
    writer = create_csv_writer(content)
    try:
      for i in xrange(batch_size):
        row = reader.next()
        writer.writerow(row)
        rows_written += 1
    except StopIteration:
      exhausted = True
    if rows_written > 0:
      yield rows_written, content.getvalue() 
开发者ID:elsigh,项目名称:browserscope,代码行数:43,代码来源:bulkload_client.py


示例12: __init__

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def __init__(self, limit):
    self.message = """
A field in your CSV input file has exceeded the current limit of %d.
You can raise this limit by adding the following lines to your config file:
import csv
csv.field_size_limit(new_limit)
where new_limit is number larger than the size in bytes of the largest
field in your CSV.
""" % limit
    Error.__init__(self, self.message) 
开发者ID:elsigh,项目名称:browserscope,代码行数:15,代码来源:bulkloader.py


示例13: __init__

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def __init__(self, istream: TextIO, sep: str = ',') -> None:
        # bump the built-in limits on field sizes
        csv.field_size_limit(2**24)
        self.reader = csv.DictReader(istream, delimiter=sep) 
开发者ID:larsyencken,项目名称:csvdiff,代码行数:7,代码来源:records.py


示例14: start

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def start(self):
        csv.field_size_limit(sys.maxsize)
        self.parser = create_parser(name=self.lang, lemmas=True) 
开发者ID:graphbrain,项目名称:graphbrain,代码行数:5,代码来源:csv_parser.py


示例15: generate_dictionary_BA

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def generate_dictionary_BA(path, files, attributes_list):
    # path = '../Minnemudac/'
    # files = ['Coborn_history_order.csv','Coborn_future_order.csv']
    # files = ['BA_history_order.csv', 'BA_future_order.csv']
    # attributes_list = ['MATERIAL_NUMBER']
    dictionary_table = {}
    counter_table = {}
    for attr in attributes_list:
        dictionary = {}
        dictionary_table[attr] = dictionary
        counter_table[attr] = 0
    csv.field_size_limit(sys.maxsize)
    for filename in files:
        count = 0
        with open(path + filename, 'r') as csvfile:
            reader = csv.reader(csvfile, delimiter=',', quotechar='|')
            for row in reader:
                if count == 0:
                    count += 1
                    continue
                key = attributes_list[0]
                if row[2] not in dictionary_table[key]:
                    dictionary_table[key][row[2]] = counter_table[key]
                    counter_table[key] = counter_table[key] + 1
                    count += 1
    print(counter_table)
    total = 0
    for key in counter_table.keys():
        total = total + counter_table[key]
    print('# dimensions of final vector: ' + str(total) + ' | ' + str(count - 1))
    return dictionary_table, total, counter_table 
开发者ID:HaojiHu,项目名称:Sets2Sets,代码行数:38,代码来源:Sets2Sets.py


示例16: csv_to_rows

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def csv_to_rows(csv_string):
  if csv_string:
    csv.field_size_limit(sys.maxsize)
    if isinstance(csv_string, str): csv_string = StringIO(csv_string)
    for row in csv.reader(csv_string, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, skipinitialspace=True, escapechar='\\'):
      yield row 
开发者ID:google,项目名称:starthinker,代码行数:8,代码来源:__init__.py


示例17: really_big_fields_enabled

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def really_big_fields_enabled(self):
        old_limit = csv.field_size_limit()
        csv.field_size_limit(2 ** 28)
        yield
        csv.field_size_limit(old_limit) 
开发者ID:datarobot,项目名称:batch-scoring,代码行数:7,代码来源:test_reader.py


示例18: read_geonames_csv

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def read_geonames_csv():
    print("Downloading geoname data from: " + GEONAMES_ZIP_URL)
    try:
        url = request.urlopen(GEONAMES_ZIP_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read()))
    print("Download complete")
    # Loading geonames data may cause errors without setting csv.field_size_limit:
    if sys.platform == "win32":
        max_c_long_on_windows = (2**32 / 2) - 1
        csv.field_size_limit(max_c_long_on_windows)
    else:
        csv.field_size_limit(sys.maxint if six.PY2 else six.MAXSIZE)
    with zipfile.open('allCountries.txt') as f:
        reader = unicodecsv.DictReader(f,
                                       fieldnames=[
                                           k for k, v in geonames_field_mappings],
                                       encoding='utf-8',
                                       delimiter='\t',
                                       quoting=csv.QUOTE_NONE)
        for d in reader:
            d['population'] = parse_number(d['population'], 0)
            d['latitude'] = parse_number(d['latitude'], 0)
            d['longitude'] = parse_number(d['longitude'], 0)
            if len(d['alternatenames']) > 0:
                d['alternatenames'] = d['alternatenames'].split(',')
            else:
                d['alternatenames'] = []
            yield d 
开发者ID:ecohealthalliance,项目名称:EpiTator,代码行数:33,代码来源:import_geonames.py


示例19: read_lcc_author_repdocs

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def read_lcc_author_repdocs(self):
        """Read and return an iterator over the author repdoc corpus, which excludes
        the authors not in the LCC.
        """
        author_repdoc_file, _, lcc_idmap_file = self.input()
        with lcc_idmap_file.open() as lcc_idmap_f:
            lcc_author_df = pd.read_csv(lcc_idmap_f, header=0, usecols=(0,))
            lcc_author_ids = lcc_author_df['author_id'].values
        csv.field_size_limit(sys.maxint)
        records = util.iter_csv_fwrapper(author_repdoc_file)
        return (doc.split('|') for author_id, doc in records
                if int(author_id) in lcc_author_ids) 
开发者ID:macks22,项目名称:dblp,代码行数:16,代码来源:repdocs.py


示例20: unicode_csv_reader

# 需要导入模块: import csv [as 别名]
# 或者: from csv import field_size_limit [as 别名]
def unicode_csv_reader(unicode_csv_data, **kwargs):
    r"""Since the standard csv library does not handle unicode in Python 2, we need a wrapper.
    Borrowed and slightly modified from the Python docs:
    https://docs.python.org/2/library/csv.html#csv-examples
    Arguments:
        unicode_csv_data: unicode csv data (see example below)
    Examples:
        >>> from torchtext.utils import unicode_csv_reader
        >>> import io
        >>> with io.open(data_path, encoding="utf8") as f:
        >>>     reader = unicode_csv_reader(f)
    """
    # Fix field larger than field limit error
    maxInt = sys.maxsize
    while True:
        # decrease the maxInt value by factor 10
        # as long as the OverflowError occurs.
        try:
            csv.field_size_limit(maxInt)
            break
        except OverflowError:
            maxInt = int(maxInt / 10)
    csv.field_size_limit(maxInt)
    for line in csv.reader(unicode_csv_data, **kwargs):
        yield line 
开发者ID:pytorch,项目名称:text,代码行数:32,代码来源:utils.py



注:本文中的csv.field_size_limit方法示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。