From 63e3fda505d2eebf89758339d5c970469f822697 Mon Sep 17 00:00:00 2001 From: Aditya Mandaleeka Date: Fri, 15 May 2015 16:56:23 -0700 Subject: [PATCH] Ensure fallback to English resources on non-Windows platforms Take the resources in RC files and build a static library that contains the ID->English string mappings. Use those strings as the key to gettext so that when gettext is not able to find a suitable string in the desired language, it falls back on returning the English string. --- src/dlls/CMakeLists.txt | 2 +- src/dlls/mscordac/CMakeLists.txt | 1 + src/dlls/mscoree/coreclr/CMakeLists.txt | 1 + src/dlls/mscorrc/CMakeLists.txt | 45 ++++++++--------- src/dlls/mscorrc/full/CMakeLists.txt | 23 ++++----- src/dlls/mscorrc/processrc.awk | 14 ++---- src/dlls/mscorrc/rctocpp.awk | 85 +++++++++++++++++++++++++++++++++ src/dlls/mscorrc/rctopo.awk | 3 +- src/dlls/mscorrc/resourcestring.h | 23 +++++++++ src/dlls/mscorrc/small/CMakeLists.txt | 23 ++++----- src/pal/inc/pal.h | 2 +- src/pal/src/locale/unicode.cpp | 33 +++++-------- src/tools/crossgen/CMakeLists.txt | 1 + src/utilcode/ccomprc.cpp | 55 ++++++++++++++++++--- 14 files changed, 220 insertions(+), 91 deletions(-) create mode 100644 src/dlls/mscorrc/rctocpp.awk create mode 100644 src/dlls/mscorrc/resourcestring.h diff --git a/src/dlls/CMakeLists.txt b/src/dlls/CMakeLists.txt index b362065..c49caa1 100644 --- a/src/dlls/CMakeLists.txt +++ b/src/dlls/CMakeLists.txt @@ -3,8 +3,8 @@ if(WIN32) endif(WIN32) if(NOT CLR_CMAKE_PLATFORM_DARWIN) add_subdirectory(dbgshim) - add_subdirectory(mscorrc) add_subdirectory(mscordbi) endif(NOT CLR_CMAKE_PLATFORM_DARWIN) add_subdirectory(mscordac) add_subdirectory(mscoree) +add_subdirectory(mscorrc) diff --git a/src/dlls/mscordac/CMakeLists.txt b/src/dlls/mscordac/CMakeLists.txt index 67f0bf6..1cb70a5 100644 --- a/src/dlls/mscordac/CMakeLists.txt +++ b/src/dlls/mscordac/CMakeLists.txt @@ -90,6 +90,7 @@ if(WIN32) ) else(WIN32) list(APPEND COREDAC_LIBRARIES + mscorrc_debug coreclrpal palrt ) diff --git a/src/dlls/mscoree/coreclr/CMakeLists.txt b/src/dlls/mscoree/coreclr/CMakeLists.txt index 17f4daf..0e6184c 100644 --- a/src/dlls/mscoree/coreclr/CMakeLists.txt +++ b/src/dlls/mscoree/coreclr/CMakeLists.txt @@ -96,6 +96,7 @@ else() ${START_WHOLE_ARCHIVE} # force all PAL objects to be included so all exports are available coreclrpal ${END_WHOLE_ARCHIVE} + mscorrc_debug palrt ) endif(WIN32) diff --git a/src/dlls/mscorrc/CMakeLists.txt b/src/dlls/mscorrc/CMakeLists.txt index c0f45f8..f307815 100644 --- a/src/dlls/mscorrc/CMakeLists.txt +++ b/src/dlls/mscorrc/CMakeLists.txt @@ -3,49 +3,46 @@ if(WIN32) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /NOENTRY") else() - set (RC_TO_PO ${CMAKE_CURRENT_SOURCE_DIR}/rctopo.awk) + set (RC_TO_CPP ${CMAKE_CURRENT_SOURCE_DIR}/rctocpp.awk) set (PROCESS_RC ${CMAKE_CURRENT_SOURCE_DIR}/processrc.awk) - - # Create a command to build gettext resources binary file from windows .rc file - # The target binary file path is returned in the variable specified by - # the TARGET_FILE parameter. + set (RESOURCE_STRING_HEADER_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + + # Create a command to create a C++ source file containing an array of + # NativeStringResource structs which represent the information from a + # given Windows .rc file. The target C++ file path is returned in the + # variable specified by the TARGET_FILE parameter. function(build_resources SOURCE TARGET_NAME TARGET_FILE) - # Ensure that the necessary tools are present find_program(AWK awk) if (AWK STREQUAL "AWK-NOTFOUND") message(FATAL_ERROR "AWK not found") endif() - find_program(MSGFMT msgfmt) - if (MSGFMT STREQUAL "MSGFMT-NOTFOUND") - message(FATAL_ERROR "msgfmt not found. Please install the gettext package.") - endif() - get_compile_definitions(PREPROCESS_DEFINITIONS) get_include_directories(INCLUDE_DIRECTORIES) set(PREPROCESSED_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.rc.i) - set(GETTEXT_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.po) - set(GETTEXT_TARGET ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.mo) + set(RESOURCE_ENTRY_ARRAY_CPP ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.cpp) add_custom_command( - OUTPUT ${GETTEXT_TARGET} + OUTPUT ${RESOURCE_ENTRY_ARRAY_CPP} # Preprocess the windows .rc file COMMAND ${CMAKE_CXX_COMPILER} -E -P ${PREPROCESS_DEFINITIONS} ${INCLUDE_DIRECTORIES} -o ${PREPROCESSED_SOURCE} -x c ${SOURCE} - # Convert the preprocessed .rc file to the .po file which is a source for the gettext toolchain - COMMAND ${AWK} -f ${RC_TO_PO} -f ${PROCESS_RC} ${PREPROCESSED_SOURCE} >${GETTEXT_SOURCE} - # Compile the .po file into the target binary .mo file - COMMAND ${MSGFMT} ${GETTEXT_SOURCE} -o ${GETTEXT_TARGET} + # Convert the preprocessed .rc file to a C++ file which will be used to make a static lib. + COMMAND ${AWK} -f ${RC_TO_CPP} -f ${PROCESS_RC} ${PREPROCESSED_SOURCE} >${RESOURCE_ENTRY_ARRAY_CPP} DEPENDS ${SOURCE} - COMMENT "Building ${TARGET_NAME}.mo" ) - - set(${TARGET_FILE} ${GETTEXT_TARGET} PARENT_SCOPE) - + + include_directories(${RESOURCE_STRING_HEADER_DIR}) + set(${TARGET_FILE} ${RESOURCE_ENTRY_ARRAY_CPP} PARENT_SCOPE) + endfunction() -endif(WIN32) +endif(WIN32) add_subdirectory(full) -add_subdirectory(small) \ No newline at end of file + +# Only add the small version of the resources if the platform is Windows. +if(WIN32) + add_subdirectory(small) +endif(WIN32) \ No newline at end of file diff --git a/src/dlls/mscorrc/full/CMakeLists.txt b/src/dlls/mscorrc/full/CMakeLists.txt index edb37d8..1e9d2d1 100644 --- a/src/dlls/mscorrc/full/CMakeLists.txt +++ b/src/dlls/mscorrc/full/CMakeLists.txt @@ -1,25 +1,22 @@ add_definitions(-DFX_VER_INTERNALNAME_STR=mscorrc.debug.dll) if(WIN32) -add_library(mscorrc.debug SHARED - ../include.rc -) -# add the install targets -install (TARGETS mscorrc.debug DESTINATION .) + add_library(mscorrc.debug SHARED + ../include.rc + ) -install (FILES ${CMAKE_CURRENT_BINARY_DIR}/$/mscorrc.debug.pdb DESTINATION PDB) + # add the install targets + install (TARGETS mscorrc.debug DESTINATION .) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/$/mscorrc.debug.pdb DESTINATION PDB) else() -build_resources(${CMAKE_CURRENT_SOURCE_DIR}/../include.rc mscorrc.debug TARGET_RESOURCE_FILE) + build_resources(${CMAKE_CURRENT_SOURCE_DIR}/../include.rc mscorrc_debug TARGET_CPP_FILE) -add_custom_target( - mscorrc.debug ALL - DEPENDS ${TARGET_RESOURCE_FILE} -) - -install (FILES ${TARGET_RESOURCE_FILE} DESTINATION en_US/LC_MESSAGES/) + add_library(mscorrc_debug STATIC + ${TARGET_CPP_FILE} + ) endif(WIN32) diff --git a/src/dlls/mscorrc/processrc.awk b/src/dlls/mscorrc/processrc.awk index 789d2c7..71279ef 100644 --- a/src/dlls/mscorrc/processrc.awk +++ b/src/dlls/mscorrc/processrc.awk @@ -33,27 +33,21 @@ BEGIN { # string starts with either a quote or L followed by a quote while (substr($i, 1, 1) != "\"" && substr($i, 1, 2) != "L\"") { - # some of the resource ids contain cast to HRESULT + # some of the resource IDs contain cast to HRESULT gsub(/\(HRESULT\)/, "", $i); - # some of the resource ids have trailing L + # some of the resource IDs have trailing L gsub(/L/, "", $i); expression = expression $i; $i=""; i++; } - # evaluate the resource id expression and format it as hex number + # evaluate the resource ID expression cmd = "echo $(("expression"))"; cmd | getline var; close(cmd); # in case shell returned the result as a string, ensure the var has numeric type var = var + 0; - # sprintf can only handle signed ints, so we need to convert - # values >= 0x80000000 to negative values - if (var >= 2147483648) - { - var = var - 4294967296; - } - var = sprintf("%X", var); + # remove the L prefix from strings gsub(/L"/, "\"", $0); # join strings "..." "..." into one diff --git a/src/dlls/mscorrc/rctocpp.awk b/src/dlls/mscorrc/rctocpp.awk new file mode 100644 index 0000000..c547bf7 --- /dev/null +++ b/src/dlls/mscorrc/rctocpp.awk @@ -0,0 +1,85 @@ +# Convert string resources from Windows native resource file to a C++ +# source file with an array of structs representing the resources. + +BEGIN { + numEntries = 0; +} + +# Takes a number and returns a string corresponding to its hex +# representation. A string representation that has fewer than 8 +# characters (not including the '0x' prefix) is padded with 0's +# to make it 8 characters. +# Example: an input of 49 yields "0x00000031". +function numberToHexString(number) +{ + quotient = number; + + hexString = ""; + while (quotient > 0) + { + remainder = quotient % 16; + quotient = int(quotient / 16); + hexString = sprintf("%x"hexString, remainder); + } + + lengthOfHexString = length(hexString); + if (lengthOfHexString < 8) + { + hexString = substr("00000000", 1, 8 - lengthOfHexString) hexString; + } + + hexString = "0x" hexString; + return hexString; +} + +# Add each entry that is in our associative array of entries to the +# C++ array we are building. The C++ array will be ordered by the +# resourceId (lowest to highest) to facilitate quick lookups. +function writesortedentries() +{ + for (entry in resourceArray) + { + # Write the entries to the C++ array ordered by the ID. + printf " {%s,%s},\n", entry, resourceArray[entry] | "sort"; + } + + # Close the pipe to ensure that the data is written now. + close("sort"); +} + +# Write entry for a string resource +# This is called for each entry. Because we want to write them in +# sorted order once all the entries have come in, for now we just +# store each entry in an associative array. +function writestringentry(id, str) +{ + numEntries++; + + # Use the string representation of the ID as the array index + # because the precision of numeric indices can be lost when + # iterating over the array in our for-in loop. + resourceArray[numberToHexString(id)] = str; +} + +# Write file header and begin the array we will populate with the resources. +function writeheader() +{ + print "// This code was generated by rctocpp.awk and is not meant to be modified manually." + print "#include "; + print ""; + print "const NativeStringResource nativeStringResources[] = {"; +} + +# Write file footer +# This function is called after all of the entries have been given to +# writestringentry. Because we know there are no more entries, we can +# now write all the entries we received so far when this is called. +# After we have written all the entries, we close the array and add a +# constant for the size of the array for convenience. +function writefooter() +{ + writesortedentries(); + print "};"; + print ""; + print "const int NUMBER_OF_NATIVE_STRING_RESOURCES = " numEntries ";"; +} diff --git a/src/dlls/mscorrc/rctopo.awk b/src/dlls/mscorrc/rctopo.awk index 32aa9ca..5229585 100644 --- a/src/dlls/mscorrc/rctopo.awk +++ b/src/dlls/mscorrc/rctopo.awk @@ -4,7 +4,8 @@ # Write entry for a string resource function writestringentry(id, str) { - print "msgid \""id"\""; + idAsStr = sprintf("%X", id); + print "msgid \""idAsStr"\""; print "msgstr" str; print ""; } diff --git a/src/dlls/mscorrc/resourcestring.h b/src/dlls/mscorrc/resourcestring.h new file mode 100644 index 0000000..475705d --- /dev/null +++ b/src/dlls/mscorrc/resourcestring.h @@ -0,0 +1,23 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +#ifndef __RESOURCE_STRING_H_ +#define __RESOURCE_STRING_H_ + +// Struct to contain a resource ID and its corresponding +// English language string. +struct NativeStringResource +{ + unsigned int resourceId; + const char* resourceString; +}; + +// Sorted array of all native string resources +extern const NativeStringResource nativeStringResources[]; + +// Number of entries in nativeStringResources +extern const int NUMBER_OF_NATIVE_STRING_RESOURCES; + +#endif // __RESOURCE_STRING_H_ diff --git a/src/dlls/mscorrc/small/CMakeLists.txt b/src/dlls/mscorrc/small/CMakeLists.txt index 75ac17c..3bd7fcd 100644 --- a/src/dlls/mscorrc/small/CMakeLists.txt +++ b/src/dlls/mscorrc/small/CMakeLists.txt @@ -1,25 +1,22 @@ add_definitions(-DFX_VER_INTERNALNAME_STR=mscorrc.dll) if(WIN32) -add_library(mscorrc SHARED - ../mscorrc.small.rc -) -# add the install targets -install (TARGETS mscorrc DESTINATION .) + add_library(mscorrc SHARED + ../mscorrc.small.rc + ) -install (FILES ${CMAKE_CURRENT_BINARY_DIR}/$/mscorrc.pdb DESTINATION PDB) + # add the install targets + install (TARGETS mscorrc DESTINATION .) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/$/mscorrc.pdb DESTINATION PDB) else() -build_resources(${CMAKE_CURRENT_SOURCE_DIR}/../mscorrc.small.rc mscorrc TARGET_RESOURCE_FILE) + build_resources(${CMAKE_CURRENT_SOURCE_DIR}/../mscorrc.small.rc mscorrc TARGET_CPP_FILE) -add_custom_target( - mscorrc ALL - DEPENDS ${TARGET_RESOURCE_FILE} -) - -install (FILES ${TARGET_RESOURCE_FILE} DESTINATION en_US/LC_MESSAGES/) + add_library(mscorrc STATIC + ${TARGET_CPP_FILE} + ) endif(WIN32) diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h index b1fb2df..8b722fb 100644 --- a/src/pal/inc/pal.h +++ b/src/pal/inc/pal.h @@ -4398,7 +4398,7 @@ int PALAPI PAL_GetResourceString( IN LPCSTR lpDomain, - IN DWORD dwResourceId, + IN LPCSTR lpResourceStr, OUT LPWSTR lpWideCharStr, IN int cchWideChar); diff --git a/src/pal/src/locale/unicode.cpp b/src/pal/src/locale/unicode.cpp index c44a6ae..6594b18 100644 --- a/src/pal/src/locale/unicode.cpp +++ b/src/pal/src/locale/unicode.cpp @@ -997,7 +997,9 @@ PAL_BindResources(IN LPCSTR lpDomain) /*++ Function : -PAL_GetResourceString - get string for a specified resource id +PAL_GetResourceString - get localized string for a specified resource. +The string that is passed in should be the English string, since it +will be returned if an appropriately localized version is not found. Returns number of characters retrieved, 0 if it failed. --*/ @@ -1005,33 +1007,22 @@ int PALAPI PAL_GetResourceString( IN LPCSTR lpDomain, - IN DWORD dwResourceId, + IN LPCSTR lpResourceStr, OUT LPWSTR lpWideCharStr, IN int cchWideChar ) { #ifndef __APPLE__ - CHAR resourceIdStr[9]; - sprintf(resourceIdStr, "%X", dwResourceId); - - // NOTE: the dgettext returns the resourceIdStr if it fails to locate - // the resource. - LPCSTR resourceString = dgettext(lpDomain, resourceIdStr); - int length = strlen(resourceString); - - return UTF8ToUnicode(resourceString, length + 1, lpWideCharStr, cchWideChar, 0); + // NOTE: dgettext returns the key if it fails to locate the appropriate + // resource. In our case, that will be the English string. + LPCSTR resourceString = dgettext(lpDomain, lpResourceStr); #else // __APPLE__ // UNIXTODO: Implement for OSX using the native localization API - // This is a temporary solution until we add the real native resource support - int len = _snwprintf(lpWideCharStr, cchWideChar - 1, W("Resource string id=0x%X"), dwResourceId); - if ((len < 0) || (len == (cchWideChar - 1))) - { - // Add string terminator if the result of the _snwprintf didn't fit the buffer. - lpWideCharStr[cchWideChar - 1] = W('\0'); - len = cchWideChar - 1; - } - - return len; + // This is a temporary solution until we add the real native resource support. + LPCSTR resourceString = lpResourceStr; #endif // __APPLE__ + + int length = strlen(resourceString); + return UTF8ToUnicode(lpResourceStr, length + 1, lpWideCharStr, cchWideChar, 0); } diff --git a/src/tools/crossgen/CMakeLists.txt b/src/tools/crossgen/CMakeLists.txt index 119e713..ceb8081 100644 --- a/src/tools/crossgen/CMakeLists.txt +++ b/src/tools/crossgen/CMakeLists.txt @@ -38,6 +38,7 @@ target_link_libraries(crossgen if(CLR_CMAKE_PLATFORM_UNIX) target_link_libraries(crossgen + mscorrc_debug coreclrpal palrt ) diff --git a/src/utilcode/ccomprc.cpp b/src/utilcode/ccomprc.cpp index 68304da..9e1cd8c 100644 --- a/src/utilcode/ccomprc.cpp +++ b/src/utilcode/ccomprc.cpp @@ -10,12 +10,15 @@ #include "ndpversion.h" #include "../dlls/mscorrc/resource.h" +#include "../dlls/mscorrc/resourcestring.h" #include "sstring.h" #include "stringarraylist.h" +#include + #ifdef USE_FORMATMESSAGE_WRAPPER // we implement the wrapper for FormatMessageW. -// Need access to the original +// Need access to the original #undef WszFormatMessage #define WszFormatMessage ::FormatMessageW #endif @@ -694,6 +697,21 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, UINT iResourceID, __out_ return LoadString(eCategory, langId, iResourceID, szBuffer, iMax, pcwchUsed); } +// Used for comparing NativeStringResource elements by ID. +int CompareNativeStringResources(const void *a, const void *b) +{ + unsigned int resourceIdA = ((NativeStringResource*)a)->resourceId; + unsigned int resourceIdB = ((NativeStringResource*)b)->resourceId; + + if (resourceIdA < resourceIdB) + return -1; + + if (resourceIdA == resourceIdB) + return 0; + + return 1; +} + HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iResourceID, __out_ecount(iMax) LPWSTR szBuffer, int iMax, int *pcwchUsed) { CONTRACTL @@ -801,7 +819,7 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iR if (szBuffer && iMax) *szBuffer = W('\0'); - length = iMax; + length = iMax; hr=HRESULT_FROM_GetLastError(); } @@ -827,7 +845,7 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iR // if we got here then we couldn't get the fallback message // the fallback message is required so just falling through into "Required" -#else // FEATURE_CORECLR +#else // FEATURE_CORECLR // everything that's not optional goes here for Desktop case DesktopCLR: case Debugging: @@ -848,7 +866,7 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iR { _ASSERTE(!"Invalid eCategory"); } - } + } } // Return an empty string to save the people with a bad error handling @@ -856,11 +874,34 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iR *szBuffer = W('\0'); return hr; -#else // !FEATURE_PAL +#else // !FEATURE_PAL int len = 0; if (szBuffer && iMax) { - len = PAL_GetResourceString(m_pResourceDomain, iResourceID, szBuffer, iMax); + // Search the sorted set of resources for the ID we're interested in. + NativeStringResource searchEntry = {iResourceID, NULL}; + NativeStringResource *resourceEntry = (NativeStringResource*)bsearch( + &searchEntry, + nativeStringResources, + NUMBER_OF_NATIVE_STRING_RESOURCES, + sizeof(NativeStringResource), + CompareNativeStringResources); + + if (resourceEntry != NULL) + { + len = PAL_GetResourceString(m_pResourceDomain, resourceEntry->resourceString, szBuffer, iMax); + } + else + { + // The resource ID wasn't found in our array. Fall back on returning the ID as a string. + len = _snwprintf(szBuffer, iMax - 1, W("[Undefined resource string ID:0x%X]"), iResourceID); + if ((len < 0) || (len == (iMax - 1))) + { + // Add string terminator if the result of _snwprintf didn't fit the buffer. + szBuffer[iMax - 1] = W('\0'); + len = iMax - 1; + } + } } if (pcwchUsed) @@ -872,7 +913,7 @@ HRESULT CCompRC::LoadString(ResourceCategory eCategory, LocaleID langId, UINT iR #endif // !FEATURE_PAL } -#ifndef DACCESS_COMPILE +#ifndef DACCESS_COMPILE HRESULT CCompRC::LoadMUILibrary(HRESOURCEDLL * pHInst) { WRAPPER_NO_CONTRACT; -- 2.7.4