mir/libreoffice
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tdf#104734 Firebird improve XClob implementation

Browse Source 
Create a more effective implementation of XClob::length() and
XClob::getSubString() methods, where string is read segment-by-segment
instead of reading the whole underlying blob. That way it is possible to
handle big texts which would not fit into memory.

Also allow reading Clob data from a resultset with getString() and
writing it in a prepared statement with setString().

Implement XPreparedStatement::setClob(). Also implement a private
version of setClob() for creating a clob from OUString:

Allow the creation of a clob column with GUI by adding a new type in
ODataBaseMetaData::getTypeInfo().

Change-Id: Ibcbbdd80e8eed5e2a3fe55b0fa196401f1bcbcdf
Reviewed-on: https://gerrit.libreoffice.org/47093
Reviewed-by: Tamás Bunth <btomi96@gmail.com>
Tested-by: Tamás Bunth <btomi96@gmail.com>
This commit is contained in:
Tamas Bunth
2017-12-13 13:46:39 +01:00
committed by Tamás Bunth
parent e7a80ddb91
commit f80b51ae44
10 changed files with 261 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
connectivity/source/drivers/firebird/Blob.cxx
View File

@@ -70,9 +70,14 @@ void Blob::ensureBlobIsOpened()
m_nBlobPosition = 0;
char aBlobItems[] = {
isc_info_blob_total_length
isc_info_blob_total_length,
isc_info_blob_max_segment
};
char aResultBuffer[20];
// Assuming a data (e.g. legth of blob) is maximum 64 bit.
// That means we need 8 bytes for data + 2 for length of data + 1 for item
// identifier for each item.
char aResultBuffer[11 + 11];
aErr = isc_blob_info(m_statusVector,
&m_blobHandle,
@@ -84,17 +89,63 @@ void Blob::ensureBlobIsOpened()
if (aErr)
evaluateStatusVector(m_statusVector, "isc_blob_info", *this);
if (*aResultBuffer == isc_info_blob_total_length)
char* pIt = aResultBuffer;
while( *pIt != isc_info_end ) // info is in clusters
{
short aResultLength = (short) isc_vax_integer(aResultBuffer+1, 2);
m_nBlobLength = isc_vax_integer(aResultBuffer+3, aResultLength);
}
else
{
assert(false);
char item = *pIt++;
short aResultLength = (short) isc_vax_integer(pIt, 2);
pIt += 2;
switch(item)
{
case isc_info_blob_total_length:
m_nBlobLength = isc_vax_integer(pIt, aResultLength);
break;
case isc_info_blob_max_segment:
m_nMaxSegmentSize = isc_vax_integer(pIt, aResultLength);
break;
default:
assert(false);
break;
}
pIt += aResultLength;
}
}
sal_uInt16 Blob::getMaximumSegmentSize()
{
ensureBlobIsOpened();
return m_nMaxSegmentSize;
}
bool Blob::readOneSegment(uno::Sequence< sal_Int8 >& rDataOut)
{
checkDisposed(Blob_BASE::rBHelper.bDisposed);
ensureBlobIsOpened();
sal_uInt16 nMaxSize = getMaximumSegmentSize();
if(rDataOut.getLength() < nMaxSize)
rDataOut.realloc(nMaxSize);
sal_uInt16 nActualSize = 0;
ISC_STATUS aRet = isc_get_segment(m_statusVector,
&m_blobHandle,
&nActualSize,
nMaxSize,
reinterpret_cast<char*>(rDataOut.getArray()) );
if (aRet && aRet != isc_segstr_eof && IndicatesError(m_statusVector))
{
OUString sError(StatusVectorToString(m_statusVector, "isc_get_segment"));
throw IOException(sError, *this);
}
m_nBlobPosition += nActualSize;
return aRet == isc_segstr_eof; // last segment read
}
void Blob::closeBlob()
{
MutexGuard aGuard(m_aMutex);

4
connectivity/source/drivers/firebird/Blob.hxx
View File

@@ -41,6 +41,7 @@ namespace connectivity
bool m_bBlobOpened;
sal_Int64 m_nBlobLength;
sal_uInt16 m_nMaxSegmentSize;
sal_Int64 m_nBlobPosition;
ISC_STATUS_ARRAY m_statusVector;
@@ -54,12 +55,15 @@ namespace connectivity
* @throws css::sdbc::SQLException
*/
void closeBlob();
sal_uInt16 getMaximumSegmentSize();
public:
Blob(isc_db_handle* pDatabaseHandle,
isc_tr_handle* pTransactionHandle,
ISC_QUAD const & aBlobID);
bool readOneSegment(css::uno::Sequence< sal_Int8 >& rDataOut);
// ---- XBlob ----------------------------------------------------
virtual sal_Int64 SAL_CALL
length() override;

86
connectivity/source/drivers/firebird/Clob.cxx
View File

@@ -28,7 +28,8 @@ Clob::Clob(isc_db_handle* pDatabaseHandle,
isc_tr_handle* pTransactionHandle,
ISC_QUAD const & aBlobID):
Clob_BASE(m_aMutex),
m_aBlob(new connectivity::firebird::Blob(pDatabaseHandle, pTransactionHandle, aBlobID))
m_aBlob(new connectivity::firebird::Blob(pDatabaseHandle, pTransactionHandle, aBlobID)),
m_nCharCount(-1)
{
}
@@ -44,13 +45,27 @@ sal_Int64 SAL_CALL Clob::length()
MutexGuard aGuard(m_aMutex);
checkDisposed(Clob_BASE::rBHelper.bDisposed);
// read the entire blob
// TODO FIXME better solution?
uno::Sequence < sal_Int8 > aEntireBlob = m_aBlob->getBytes( 1, m_aBlob->length());
OUString sEntireClob ( reinterpret_cast< sal_Char *>( aEntireBlob.getArray() ),
aEntireBlob.getLength(),
if( m_nCharCount >= 0 )
return m_nCharCount;
m_nCharCount = 0;
// Read each segment, and calculate it's size by interpreting it as a
// character stream. Assume that no characters are split by the segments.
bool bLastSegmRead = false;
do
{
uno::Sequence < sal_Int8 > aSegmentBytes;
bLastSegmRead = m_aBlob->readOneSegment( aSegmentBytes );
OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
aSegmentBytes.getLength(),
RTL_TEXTENCODING_UTF8 );
return sEntireClob.getLength();
if( !bLastSegmRead)
m_nCharCount += sSegment.getLength();
}while( !bLastSegmRead );
m_aBlob->closeInput(); // reset position
return m_nCharCount;
}
OUString SAL_CALL Clob::getSubString(sal_Int64 nPosition,
@@ -58,19 +73,58 @@ OUString SAL_CALL Clob::getSubString(sal_Int64 nPosition,
{
MutexGuard aGuard(m_aMutex);
checkDisposed(Clob_BASE::rBHelper.bDisposed);
// TODO do not reset position if it is not necessary
m_aBlob->closeInput(); // reset position
// read the entire blob
// TODO FIXME better solution?
// TODO FIXME Assume indexing of nPosition starts at position 1.
uno::Sequence < sal_Int8 > aEntireBlob = m_aBlob->getBytes( 1, m_aBlob->length());
OUString sEntireClob ( reinterpret_cast< sal_Char *>( aEntireBlob.getArray() ),
aEntireBlob.getLength(),
OUStringBuffer sSegmentBuffer;
sal_Int64 nActPos = 1;
sal_Int32 nActLen = 0;
// skip irrelevant parts
while( nActPos < nPosition )
{
uno::Sequence < sal_Int8 > aSegmentBytes;
bool bLastRead = m_aBlob->readOneSegment( aSegmentBytes );
if( bLastRead )
throw lang::IllegalArgumentException("nPosition out of range", *this, 0);
OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
aSegmentBytes.getLength(),
RTL_TEXTENCODING_UTF8 );
sal_Int32 nStrLen = sSegment.getLength();
nActPos += nStrLen;
if( nActPos > nPosition )
{
sal_Int32 nCharsToCopy = static_cast<sal_Int32>(nActPos - nPosition);
if( nCharsToCopy > nLength )
nCharsToCopy = nLength;
// append relevant part of first segment
sSegmentBuffer.append( sSegment.copy(0, nCharsToCopy ) );
nActLen += sSegmentBuffer.getLength();
}
}
if( nPosition + nLength - 1 > sEntireClob.getLength() )
throw lang::IllegalArgumentException("nPosition out of range", *this, 0);
// read nLength characters
while( nActLen < nLength )
{
uno::Sequence < sal_Int8 > aSegmentBytes;
bool bLastRead = m_aBlob->readOneSegment( aSegmentBytes );
return sEntireClob.copy(nPosition - 1 , nLength);
OUString sSegment ( reinterpret_cast< sal_Char *>( aSegmentBytes.getArray() ),
aSegmentBytes.getLength(),
RTL_TEXTENCODING_UTF8 );
sal_Int32 nStrLen = sSegment.getLength();
if( nActLen + nStrLen > nLength )
sSegmentBuffer.append(sSegment.copy(0, nLength - nActLen) );
else
sSegmentBuffer.append(sSegment);
nActLen += nStrLen;
if( bLastRead && nActLen < nLength )
throw lang::IllegalArgumentException("out of range", *this, 0);
}
return sSegmentBuffer.makeStringAndClear();
}
uno::Reference< XInputStream > SAL_CALL Clob::getCharacterStream()

2
connectivity/source/drivers/firebird/Clob.hxx
View File

@@ -38,6 +38,8 @@ namespace connectivity
*/
rtl::Reference<connectivity::firebird::Blob> m_aBlob;
sal_Int64 m_nCharCount;
public:
Clob(isc_db_handle* pDatabaseHandle,
isc_tr_handle* pTransactionHandle,

9
connectivity/source/drivers/firebird/DatabaseMetaData.cxx
View File

@@ -870,6 +870,15 @@ uno::Reference< XResultSet > SAL_CALL ODatabaseMetaData::getTypeInfo()
aRow[6] = new ORowSetValueDecorator(OUString("length")); // Create Params
aRow[9] = new ORowSetValueDecorator(
sal_Int16(ColumnSearch::NONE)); // Searchable
// Clob (SQL_BLOB)
aRow[1] = new ORowSetValueDecorator(OUString("BLOB")); // BLOB, with subtype 1
aRow[2] = new ORowSetValueDecorator(DataType::CLOB);
aRow[3] = new ORowSetValueDecorator(sal_Int16(2147483647)); // Precision = max length
aRow[6] = new ORowSetValueDecorator(); // Create Params
aRow[9] = new ORowSetValueDecorator(
sal_Int16(ColumnSearch::FULL)); // Searchable
aRow[12] = new ORowSetValueDecorator(false); // Autoincrement
aRow[14] = ODatabaseMetaDataResultSet::get0Value(); // Minimum scale
aRow[15] = ODatabaseMetaDataResultSet::get0Value(); // Max scale
aResults.push_back(aRow);

102
connectivity/source/drivers/firebird/PreparedStatement.cxx
View File

@@ -178,10 +178,10 @@ void SAL_CALL OPreparedStatement::disposing()
}
void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
const OUString& x)
const OUString& sInput)
{
SAL_INFO("connectivity.firebird",
"setString(" << nParameterIndex << " , " << x << ")");
"setString(" << nParameterIndex << " , " << sInput << ")");
MutexGuard aGuard( m_aMutex );
checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
@@ -190,7 +190,7 @@ void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
checkParameterIndex(nParameterIndex);
setParameterNull(nParameterIndex, false);
OString str = OUStringToOString(x , RTL_TEXTENCODING_UTF8 );
OString str = OUStringToOString(sInput , RTL_TEXTENCODING_UTF8 );
XSQLVAR* pVar = m_pInSqlda->sqlvar + (nParameterIndex - 1);
@@ -219,6 +219,10 @@ void SAL_CALL OPreparedStatement::setString(sal_Int32 nParameterIndex,
// Fill remainder with spaces
memset(pVar->sqldata + str.getLength(), ' ', pVar->sqllen - str.getLength());
break;
case SQL_BLOB: // Clob
assert( pVar->sqlsubtype == static_cast<short>(BlobSubtype::Clob) );
setClob(nParameterIndex, sInput );
break;
default:
::dbtools::throwSQLException(
"Incorrect type for setString",
@@ -504,21 +508,105 @@ void OPreparedStatement::closeBlobAfterWriting(isc_blob_handle& rBlobHandle)
ISC_STATUS aErr;
aErr = isc_close_blob(m_statusVector,
&rBlobHandle);
&rBlobHandle);
if (aErr)
{
evaluateStatusVector(m_statusVector,
"isc_close_blob failed",
*this);
"isc_close_blob failed",
*this);
assert(false);
}
}
void SAL_CALL OPreparedStatement::setClob( sal_Int32, const Reference< XClob >& )
void SAL_CALL OPreparedStatement::setClob(sal_Int32 nParameterIndex, const Reference< XClob >& xClob )
{
::osl::MutexGuard aGuard( m_aMutex );
checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
#if SAL_TYPES_SIZEOFPOINTER == 8
isc_blob_handle aBlobHandle = 0;
#else
isc_blob_handle aBlobHandle = nullptr;
#endif
ISC_QUAD aBlobId;
openBlobForWriting(aBlobHandle, aBlobId);
// Max segment size is 2^16 == SAL_MAX_UINT16
// SAL_MAX_UINT16 / 4 is surely enough for UTF-8
// TODO apply max segment size to character encoding
sal_Int64 nCharWritten = 1; // XClob is indexed from 1
ISC_STATUS aErr = 0;
sal_Int64 nLen = xClob->length();
while ( nLen > nCharWritten )
{
sal_Int64 nCharRemain = nLen - nCharWritten;
constexpr sal_uInt16 MAX_SIZE = SAL_MAX_UINT16 / 4;
sal_uInt16 nWriteSize = (nCharRemain > MAX_SIZE) ? MAX_SIZE : nCharRemain;
OString sData = OUStringToOString(
xClob->getSubString(nCharWritten, nWriteSize),
RTL_TEXTENCODING_UTF8);
aErr = isc_put_segment( m_statusVector,
&aBlobHandle,
sData.getLength(),
sData.getStr() );
nCharWritten += nWriteSize;
if (aErr)
break;
}
// We need to make sure we close the Blob even if their are errors, hence evaluate
// errors after closing.
closeBlobAfterWriting(aBlobHandle);
if (aErr)
{
evaluateStatusVector(m_statusVector,
"isc_put_segment failed",
*this);
assert(false);
}
setValue< ISC_QUAD >(nParameterIndex, aBlobId, SQL_BLOB);
}
void OPreparedStatement::setClob( sal_Int32 nParameterIndex, const OUString& rStr )
{
::osl::MutexGuard aGuard( m_aMutex );
checkDisposed(OStatementCommonBase_Base::rBHelper.bDisposed);
#if SAL_TYPES_SIZEOFPOINTER == 8
isc_blob_handle aBlobHandle = 0;
#else
isc_blob_handle aBlobHandle = nullptr;
#endif
ISC_QUAD aBlobId;
openBlobForWriting(aBlobHandle, aBlobId);
OString sData = OUStringToOString(
rStr,
RTL_TEXTENCODING_UTF8);
ISC_STATUS aErr = isc_put_segment( m_statusVector,
&aBlobHandle,
sData.getLength(),
sData.getStr() );
// We need to make sure we close the Blob even if their are errors, hence evaluate
// errors after closing.
closeBlobAfterWriting(aBlobHandle);
if (aErr)
{
evaluateStatusVector(m_statusVector,
"isc_put_segment failed",
*this);
assert(false);
}
setValue< ISC_QUAD >(nParameterIndex, aBlobId, SQL_BLOB);
}
void SAL_CALL OPreparedStatement::setBlob(sal_Int32 nParameterIndex,

1
connectivity/source/drivers/firebird/PreparedStatement.hxx
View File

@@ -78,6 +78,7 @@ namespace connectivity
* Assumes that all necessary mutexes have been taken.
*/
void closeBlobAfterWriting(isc_blob_handle& rBlobHandle);
void setClob(sal_Int32 nParamIndex, const OUString& rStr);
protected:
virtual void SAL_CALL setFastPropertyValue_NoBroadcast(sal_Int32 nHandle,

5
connectivity/source/drivers/firebird/ResultSet.cxx
View File

@@ -604,6 +604,11 @@ OUString OResultSet::retrieveValue(const sal_Int32 nColumnIndex, const ISC_SHORT
return OUString(); // never reached
}
}
else if(aSqlType == SQL_BLOB && aSqlSubType == static_cast<short>(BlobSubtype::Clob) )
{
uno::Reference<XClob> xClob = getClob(nColumnIndex);
return xClob->getSubString( 0, xClob->length() );
}
else
{
return retrieveValue< ORowSetValue >(nColumnIndex, 0);

7
connectivity/source/drivers/firebird/Tables.cxx
View File

@@ -106,6 +106,13 @@ OUString Tables::createStandardColumnPart(const Reference< XPropertySet >& xColP
aSql.append(" ");
aSql.append("CHARACTER SET OCTETS");
}
else if(aType == DataType::CLOB)
{
// CLOB is a special type of blob in Firebird context.
// Subtype number 1 always refers to CLOB
aSql.append(" ");
aSql.append("SUB_TYPE 1");
}
}
if ( bIsAutoIncrement && !sAutoIncrementValue.isEmpty())

15
connectivity/source/drivers/firebird/Util.cxx
View File

@@ -118,13 +118,14 @@ sal_Int32 firebird::ColumnTypeInfo::getSdbcType() const
short aSubType = m_aSubType;
if( m_nScale > 0 )
{
// scale makes sense only for decimal and numeric types
assert(aType == SQL_SHORT || aType == SQL_LONG || aType == SQL_DOUBLE
|| aType == SQL_INT64);
// if scale is set without subtype then imply numeric
if( static_cast<NumberSubType>(aSubType) == NumberSubType::Other )
aSubType = static_cast<short>(NumberSubType::Numeric);
// numeric / decimal
if(aType == SQL_SHORT || aType == SQL_LONG || aType == SQL_DOUBLE
|| aType == SQL_INT64)
{
// if scale is set without subtype then imply numeric
if( static_cast<NumberSubType>(aSubType) == NumberSubType::Other )
aSubType = static_cast<short>(NumberSubType::Numeric);
}
}
switch (aType)