Proper unit testing of iOS applications with multiple schemes

I have recently faced a strange problem in the iOS application we develop. It turned out we were not able to build the app in test configuration for running unit tests in one of our two production schemes. Xcodebuild was complaining about linking the storyboard, Image.xcassets, localization files and one of the resource bundles. The first three errors didn’t contain any meaningful descriptions, but the fourth one was telling that it cannot copy the bundle as the target directory is not empty. I googled this error, but didn’t find any quick fixes that would work. So I started to dig in.

That’s how the error looked like.

[1mFailures:[0m
 
  0) Copy /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/CountryFlags.bundle
[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[2mCpResource MyAwesomeApp/Resources/CountryFlags.bundle build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/CountryFlags.bundle
    cd /Users/zmicier/Jenkins/workspace/failing_build
    export PATH=”/Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode_6.4.app/Contents/Developer/usr/bin:/Users/zmicier/.rbenv/shims:/Users/zmicier/Software/git-subrepo/lib:/usr/bin:/bin:/usr/sbin:/sbin”
    builtin-copy -exclude .DS_Store -exclude CVS -exclude .svn -exclude .git -exclude .hg -resolve-src-symlinks /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/CountryFlags.bundle /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app
 
error: remove /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/CountryFlags.bundle: Directory not empty
error: couldn’t remove ‘/Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/CountryFlags.bundle’ after command failed: Directory not empty[0m[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
 
  1) Compile asset catalog /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/Images.xcassets
[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[2mCompileAssetCatalog build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app MyAwesomeApp/Resources/Images.xcassets MyAwesomeAppSomeCloud/Resources/SomeCloud.xcassets
    cd /Users/zmicier/Jenkins/workspace/failing_build
    export PATH=”/Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode_6.4.app/Contents/Developer/usr/bin:/Users/zmicier/.rbenv/shims:/Users/zmicier/Software/git-subrepo/lib:/usr/bin:/bin:/usr/sbin:/sbin”
    /Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin/actool –output-format human-readable-text –notices –warnings –export-dependency-info /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Intermediates/MyAwesomeApp.build/Test-iphonesimulator/MyAwesomeApp.build/assetcatalog_dependencies.txt –output-partial-info-plist /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Intermediates/MyAwesomeApp.build/Test-iphonesimulator/MyAwesomeApp.build/assetcatalog_generated_info.plist –app-icon AppIcon –launch-image LaunchImage –platform iphonesimulator –minimum-deployment-target 7.1 –target-device iphone –target-device ipad –compress-pngs –compile /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/Images.xcassets /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeAppSomeCloud/Resources/SomeCloud.xcassets
 
[0m[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
 
  2) Compile Storyboard file /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/Base.lproj/MainStoryboard.storyboard
[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[2mCompileStoryboard MyAwesomeApp/Resources/Base.lproj/MainStoryboard.storyboard
    cd /Users/zmicier/Jenkins/workspace/failing_build
    export PATH=”/Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode_6.4.app/Contents/Developer/usr/bin:/Users/zmicier/.rbenv/shims:/Users/zmicier/Software/git-subrepo/lib:/usr/bin:/bin:/usr/sbin:/sbin”
    export XCODE_DEVELOPER_USR_PATH=/Applications/Xcode_6.4.app/Contents/Developer/usr/bin/..
    /Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin/ibtool –target-device iphone –target-device ipad –errors –warnings –notices –module MyAwesomeApp –minimum-deployment-target 7.1 –output-partial-info-plist /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Intermediates/MyAwesomeApp.build/Test-iphonesimulator/MyAwesomeApp.build/MainStoryboard-SBPartialInfo.plist –auto-activate-custom-fonts –output-format human-readable-text –compilation-directory /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/Base.lproj /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/Base.lproj/MainStoryboard.storyboard
 
[0m[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
 
  3) Copy /Users/zmicier/Jenkins/workspace/failing_build/MyAwesomeApp/Resources/es-MX.lproj/Localizable.strings
[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
[2mCopyStringsFile build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/es-MX.lproj/Localizable.strings MyAwesomeApp/Resources/es-MX.lproj/Localizable.strings
    cd /Users/zmicier/Jenkins/workspace/failing_build
    export PATH=”/Applications/Xcode_6.4.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode_6.4.app/Contents/Developer/usr/bin:/Users/zmicier/.rbenv/shims:/Users/zmicier/Software/git-subrepo/lib:/usr/bin:/bin:/usr/sbin:/sbin”
    builtin-copyStrings –validate –inputencoding utf-8 –outputencoding binary –outdir /Users/zmicier/Jenkins/workspace/failing_build/build/Test/Build/Products/Test-iphonesimulator/MyAwesomeApp.app/es-MX.lproj — MyAwesomeApp/Resources/es-MX.lproj/Localizable.strings
 
[0m[2m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m
To give you an understanding of this error and the root cause of it, let me first explain our setup. We work on an iOS application that was initially developed by another team. This is an Objective-C app with a storyboard and a number of modules. It has an application module with all the views and controllers, some backend modules with different responsibilities and test modules with unit tests covering the application logic. We release two slightly different versions of the app for different markets so the project has two production schemes to build these different flavours (let’s call them A and B).
When we took control over the app, it turned out that the development processes were not well maintained and we’d spent quite some time to put this project on track. We noticed that the previous development team was only running unit tests for flavour A, so we added CI jobs to also test the second flavour. However, due to the lack of CI machines, we weren’t running the unit tests for the flavour B regularly. After we got the new hardware and configured the tests for flavour B to run every day, the respective builds failed to compile. Strangely, I was able to run the builds at my local MacBook, but the MacMinis used as Jenkins slaves were showing me linker errors. Well, sometimes I was able to build the app there, but in 95% of my attempts I saw the error.
After I played a while with the problematic resource bundle and didn’t find anything wrong about it, I decided to find a commit introducing this weird behaviour. I remembered that we didn’t have this problem when I first configured the builds, so I was pretty sure it’s something introduced recently. Imagine how surprised I was when I found out that this problem started to happen after we changed the settings of location permission request.
Our app requires access to the user’s location but doesn’t need in these data when the app is in background. However, for some reasons we were asking the users to give us location access all the time, not only when the app is being used. The fine-grained control over this setting was introduced in iOS 8 and I suspect that the developer who updated the app for the new SDK just didn’t bother with restricting the app permissions. I’ve changed the code by altering settings in info.plist and replacing requestAlwaysAuthorization to requestWhenInUseAuthorization in this code snippet.
if ([self.locationManager respondsToSelector:@selector(requestAlwaysAuthorization)])
{
   [self.locationManager requestAlwaysAuthorization];
}
And that was it. After this change, the tests started to fail. By the way, quite recently we’ve migrated the tests from using iOS Simulator 7 to iOS Simulator 8 with significant CoreLocation updates and when I tried to build the tests for iOS Simulator 7, the error was obviously gone, as the code inside ifdef was never triggered. But we wanted to make it working with iOS Simulator 8, so I kept investigating.
Long story short, I found the cause of the problem in the project file. When we build the app for the unit tests, we need to configure TEST_HOST and BUNDLE_LOADER settings. The former configures the host application that has to be loaded for tests injection and the latter points to the loader of resource bundles. If your tests don’t need to access the resources of the app or its code, you don’t have to configure these variables, but if they do you have to set them up properly. It turned out that in our app these settings were configured to always point to flavour A, thus when building the flavour B we still had to first build the flavour A, and this configuration was introducing the conflicts. So, if you have several production schemes that you want to test, you need to configure TEST_HOST and BUNDLE_LOADER dynamically and that’s how you do it.
First of all, you need to find the paths of the bundle loader for all your app flavours. In our case, the paths were the following.
"$(BUILT_PRODUCTS_DIR)/MyAwesomeApp.app/AMyAwesomeApp"
"$(BUILT_PRODUCTS_DIR)/MyAwesomeApp.app/BMyAwesomeApp"
As you can see, they differ by flavour prefix in the file name. This prefix has to be configured with a variable that is available to xcodebuild when it parses the build settings. This means that you cannot just add this variable to your xcconfig files or environment variables in scheme settings. The only way I managed to make this working was by adding this variable as a command-line argument to xctool/xcodebuild. For us this is an acceptable solution as we are used to run the unit tests from command line, but if you want to run the tests dynamically from Xcode, this wouldn’t work.
So, I’ve changed TEST_HOST and BUNDLE_LOADER to contain the following value:
"$(BUILT_PRODUCTS_DIR)/MyAwesomeApp.app/$(APP_FLAVOR)MyAwesomeApp"

To propagate this value to all test modules, I added the following post-installer to the Podfile.

# Adds TEST_HOST variable for Test configurations of unit test modules for dynamic bundle loader resolution depending on the flavor being tested (A/B).
#
# APP_FLAVOR variable has to be defined as a command line argument to xcodebuild/xctool command running the tests.
post_install do |installer|
    wd = Dir.pwd
    installer.pods_project.targets.each do |target|
        if target.name.include? "Tests"
            target.build_configurations.each do |config|
                if config.name.include? "Test"
                    xcConfigFilename = "#{wd}/Pods/Target Support Files/#{target.name}/#{target.name}.#{config.name.downcase}.xcconfig"
                    xcConfig = File.read(xcConfigFilename)
                    xcConfig << "\nTEST_HOST=$(BUILT_PRODUCTS_DIR)/MyAwesomeApp.app/$(APP_FLAVOR)MyAwesomeApp"
                    File.open(xcConfigFilename, "w") { |file| file << xcConfig }
                end
            end
        end
    end
end

When you add this post-installer to the Podfile and run pod install, all the xcconfig files generated by Cocoapods will have TEST_HOST variable containing the path to a resource bundle loader with a placeholder $(APP_FLAVOR) instead of a real flavour. Next, you have to update TEST_HOST and BUNDLE_LOADER in your project settings for the test modules to fetch this value from the generated xcconfig files (or from plain xcconfig files if you don’t use Cocoapods). And eventually you have to run xcodebuild or xctool with APP_FLAVOR=A or APP_FLAVOR=B command-line argument for tests to link correctly.

Advertisements

Пакінуць адказ

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Змяніць )

Twitter picture

You are commenting using your Twitter account. Log Out / Змяніць )

Facebook photo

You are commenting using your Facebook account. Log Out / Змяніць )

Google+ photo

You are commenting using your Google+ account. Log Out / Змяніць )

Connecting to %s