pybiomart sqlite3.InterfaceError

Question

[ngarber]

New error... I'm trying to make a big query to the hsapiens_gene_ensembl dataset, and it appears to be very angry with me! Does anyone know what this error means and/or how to fix it?

import pybiomart

server = pybiomart.Server(host = "http://www.ensembl.org")
mart = server["ENSEMBL_MART_ENSEMBL"]
dataset = mart["hsapiens_gene_ensembl"]

data_df = dataset.query(attributes = ['ensembl_gene_id', 'ensembl_transcript_id', 'ensembl_peptide_id', 'external_gene_name', 'mmusculus_homolog_ensembl_gene', 'mmusculus_homolog_associated_gene_name', 'mmusculus_homolog_ensembl_peptide', 'dmelanogaster_homolog_ensembl_gene', 'dmelanogaster_homolog_associated_gene_name', 'dmelanogaster_homolog_ensembl_peptide', 'celegans_homolog_ensembl_gene', 'celegans_homolog_associated_gene_name', 'celegans_homolog_ensembl_peptide', 'drerio_homolog_ensembl_gene', 'drerio_homolog_associated_gene_name', 'drerio_homolog_ensembl_peptide']

Traceback (most recent call last):
  File "/home/tiltwolf/Documents/GitHub/PACM/Step7_SLiM_Conservation.py", line 89, in <module>
    biomart_homolog_df = dataset.query(attributes = query_attributes)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/dataset.py", line 262, in query
    response = self.get(query=ElementTree.tostring(root))
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/base.py", line 107, in get
    r = requests.get(self.url, params=params)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 115, in request
    return super().request(method, url, *args, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 147, in send
    response = self._send_and_cache(request, actions, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 193, in _send_and_cache
    self.cache.save_response(response, actions.cache_key, actions.expires)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/base.py", line 97, in save_response
    self.responses[cache_key] = cached_response
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/sqlite.py", line 268, in __setitem__
    super().__setitem__(key, serialized_value)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/sqlite.py", line 220, in __setitem__
    con.execute(
sqlite3.InterfaceError: Error binding parameter 1 - probably unsupported type.
  File "/home/tiltwolf/Documents/GitHub/PACM/Step7_SLiM_Conservation.py", line 89, in <module>
    biomart_homolog_df = dataset.query(attributes = query_attributes)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/dataset.py", line 262, in query
    response = self.get(query=ElementTree.tostring(root))
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/base.py", line 107, in get
    r = requests.get(self.url, params=params)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 115, in request
    return super().request(method, url, *args, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 147, in send
    response = self._send_and_cache(request, actions, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 193, in _send_and_cache
    self.cache.save_response(response, actions.cache_key, actions.expires)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/base.py", line 97, in save_response
    self.responses[cache_key] = cached_response
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/sqlite.py", line 268, in __setitem__
    super().__setitem__(key, serialized_value)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/backends/sqlite.py", line 220, in __setitem__
    con.execute(
sqlite3.InterfaceError: Error binding parameter 1 - probably unsupported type.

Answer 1

[ngarber]

I tried requesting less fields and only getting the critical ones for my application, and now I'm getting a new error (requests.exceptions.ChunkedEncodingError). Help!

import pybiomart

server = pybiomart.Server(host = "http://www.ensembl.org")
mart = server["ENSEMBL_MART_ENSEMBL"]
dataset = mart["hsapiens_gene_ensembl"]

data_df = dataset.query(attributes = ['ensembl_peptide_id', 'external_gene_name', 'mmusculus_homolog_ensembl_peptide', 'rnorvegicus_homolog_ensembl_peptide', 'dmelanogaster_homolog_ensembl_peptide', 'celegans_homolog_ensembl_peptide', 'drerio_homolog_ensembl_peptide']
Traceback (most recent call last):
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 697, in _update_chunk_length
    self.chunk_left = int(line, 16)
ValueError: invalid literal for int() with base 16: b''

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 438, in _error_catcher
    yield
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 764, in read_chunked
    self._update_chunk_length()
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 701, in _update_chunk_length
    raise InvalidChunkLength(self, line)
urllib3.exceptions.InvalidChunkLength: InvalidChunkLength(got length b'', 0 bytes read)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/site-packages/requests/models.py", line 760, in generate
    for chunk in self.raw.stream(chunk_size, decode_content=True):
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 572, in stream
    for line in self.read_chunked(amt, decode_content=decode_content):
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 752, in read_chunked
    with self._error_catcher():
  File "/usr/lib64/python3.10/contextlib.py", line 153, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/lib/python3.10/site-packages/urllib3/response.py", line 455, in _error_catcher
    raise ProtocolError("Connection broken: %r" % e, e)
urllib3.exceptions.ProtocolError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/tiltwolf/Documents/GitHub/PACM/Step7_SLiM_Conservation.py", line 94, in <module>
    biomart_homolog_df = dataset.query(attributes = query_attributes)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/dataset.py", line 262, in query
    response = self.get(query=ElementTree.tostring(root))
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/pybiomart/base.py", line 107, in get
    r = requests.get(self.url, params=params)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/api.py", line 61, in request
    return session.request(method=method, url=url, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 115, in request
    return super().request(method, url, *args, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 147, in send
    response = self._send_and_cache(request, actions, **kwargs)
  File "/home/tiltwolf/.local/lib/python3.10/site-packages/requests_cache/session.py", line 189, in _send_and_cache
    response = super().send(request, **kwargs)
  File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 687, in send
    r.content
  File "/usr/lib/python3.10/site-packages/requests/models.py", line 838, in content
    self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
  File "/usr/lib/python3.10/site-packages/requests/models.py", line 763, in generate
    raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ("Connection broken: InvalidChunkLength(got length b'', 0 bytes read)", InvalidChunkLength(got length b'', 0 bytes read))

Answer 2

[zorbax]

As I mentioned in an answer to your previous question, you should initiate an instance and then use a method:

from pybiomart import Server

server = Server(host='http://www.ensembl.org')

dataset = server.marts['ENSEMBL_MART_ENSEMBL']['hsapiens_gene_ensembl']

Then your query will work:

dataset.query(attributes=['ensembl_gene_id', 'ensembl_transcript_id', 'ensembl_peptide_id', 'external_gene_name', 
                          'mmusculus_homolog_ensembl_gene', 'mmusculus_homolog_associated_gene_name',
                          'mmusculus_homolog_ensembl_peptide', 'dmelanogaster_homolog_ensembl_gene', 
                          'dmelanogaster_homolog_associated_gene_name', 'dmelanogaster_homolog_ensembl_peptide', 
                          'celegans_homolog_ensembl_gene', 'celegans_homolog_associated_gene_name', 
                          'celegans_homolog_ensembl_peptide', 'drerio_homolog_ensembl_gene', 
                          'drerio_homolog_associated_gene_name', 'drerio_homolog_ensembl_peptide']

Use server.marts['ENSEMBL_MART_ENSEMBL'] instead server['ENSEMBL_MART_ENSEMBL']