Fix non-native full screen misc background color and transparency issues

Fixed misc bugs:

- Previously, setting a colorscheme while in non-native full screen would
override the background color specified in 'fuoptions' (default is
black). It also wouldn't update the specified color if it's specified to
a highlight group (e.g. `set fuopts=background:Normal`).

- Transparency and background colors and setting colorscheme also didn't
work well togehter. Changing a background color while in fullscreen
would reset transparency back to 0.

Also, 'fuoptions' can now specify background color in 6-digit hex number
now, in addition to the old 8-digit format. Also update docs to
elaborate that when using 8-digit, the alpha component is ignored (since
we use the 'transparency' setting instead).

Add new tests to test 'fuoptions' work.

Author: ychin

runtime/doc/options.txt

@@ -3981,15 +3981,19 @@ A jump table for the options with a short description can be found at |Q_op|.
 		of columns fitting on the screen in fullscreen mode. If unset,
 		'columns' will be unchanged when entering fullscreen mode.
 	background:color
-		When entering fullscreen, 'color' defines the color of the part
-		of the screen that is not occupied by the Vim control. If
-		'color' is an 8-digit hexadecimal number preceded by '#',
-		it is interpreted as an explicit color value '#aarrggbb', with
-		one byte each for the alpha, red, green, and blue values.
-		Otherwise, 'color' is interpreted as a highlight group name,
+		When entering fullscreen, "color" defines the color of the part
+		of the screen that is not occupied by the Vim control,
+		including the "notch" area for MacBook laptops.
+		If "color" is a 6 or 8-digit hexadecimal number preceded by
+		'#', it is interpreted as an explicit color value '#rrggbb' /
+		'#aarrggbb', with one byte each for the alpha, red, green, and
+		blue values. The alpha value is ignored (use 'transparency'
+		instead for a translucent window).
+		Otherwise, "color" is interpreted as a highlight group name,
 		and the fullscreen background is filled with that highlight
 		group's background color, as defined by the current color
 		scheme.
+		If unset, the default value is black (#FF000000).
 
 	Examples:
 	Don't change size when entering fullscreen: >

src/MacVim/MMBackend.m

@@ -1188,7 +1188,7 @@ - (void)leaveFullScreen
 - (void)setFullScreenBackgroundColor:(int)color
 {
     NSMutableData *data = [NSMutableData data];
-    color = MM_COLOR(color);
+    color = MM_COLOR_WITH_TRANSP(color,p_transp);
     [data appendBytes:&color length:sizeof(int)];
 
     [self queueMessage:SetFullScreenColorMsgID data:data];

src/MacVim/MMVimController.m

@@ -1129,7 +1129,7 @@ - (void)handleMessage:(int)msgid data:(NSData *)data
         case SetFullScreenColorMsgID:
         {
             const int *bg = (const int*)[data bytes];
-            NSColor *color = [NSColor colorWithRgbInt:*bg];
+            NSColor *color = [NSColor colorWithArgbInt:*bg];
 
             [windowController setFullScreenBackgroundColor:color];
         }

src/MacVim/MMWindowController.m

@@ -699,25 +699,32 @@ - (void)setDefaultColorsBackground:(NSColor *)back foreground:(NSColor *)fore
         // window, so we need to set a transparency color here to make the
         // transparency show through.
         if ([back alphaComponent] == 1) {
-            // Here, any solid color would do, but setting it with "back" has an
-            // interesting effect where the title bar gets subtly tinted by it
-            // as well, so do that. (Note that this won't play well in <=10.12
-            // since we are using the deprecated
-            // NSWindowStyleMaskTexturedBackground which makes the titlebars
-            // transparent in those. Consider not using textured background.)
+            // The window's background color affects the title bar tint and
+            // if we are using a transparent title bar this color will show
+            // up as well.
+            // (Note that this won't play well in <=10.12 since we are using
+            // the deprecated NSWindowStyleMaskTexturedBackground which makes
+            // the titlebars transparent in those. Consider not using textured
+            // background.)
             [decoratedWindow setBackgroundColor:back];
+
+            // Note: We leave the full screen window's background color alone
+            // because it is affected by 'fuoptions' instead. We just change the
+            // alpha back to 1 in case it was changed previously because transparency
+            // was set.
             if (fullScreenWindow) {
-                [fullScreenWindow setBackgroundColor:back];
+                [fullScreenWindow setBackgroundColor:
+                 [fullScreenWindow.backgroundColor colorWithAlphaComponent:1]];
             }
         } else {
             // HACK! We really want a transparent background color to avoid
             // double blending the transparency, but setting alpha=0 leads to
             // the window border disappearing and also drag-to-resize becomes a
             // lot slower. So hack around it by making it virtually transparent.
-            NSColor *clearColor = [back colorWithAlphaComponent:0.001];
-            [decoratedWindow setBackgroundColor:clearColor];
+            [decoratedWindow setBackgroundColor:[back colorWithAlphaComponent:0.001]];
             if (fullScreenWindow) {
-                [fullScreenWindow setBackgroundColor:clearColor];
+                [fullScreenWindow setBackgroundColor:
+                 [fullScreenWindow.backgroundColor colorWithAlphaComponent:0.001]];
             }
         }
     }
@@ -1046,9 +1053,17 @@ - (void)leaveFullScreen
     }
 }
 
+/// Called when the window is in non-native full-screen mode and the user has
+/// updated the background color.
 - (void)setFullScreenBackgroundColor:(NSColor *)back
 {
     if (fullScreenWindow)
+        // See setDefaultColorsBackground: for why set a transparent
+        // background color, and why 0.001 instead of 0.
+        if ([back alphaComponent] != 1) {
+            back = [back colorWithAlphaComponent:0.001];
+        }
+
         [fullScreenWindow setBackgroundColor:back];
 }
 

src/MacVim/MacVimTests/MacVimTests.m

@@ -737,6 +737,8 @@ - (void) testWindowResize {
     [self waitForVimClose];
 }
 
+#pragma mark Full screen tests
+
 - (void)waitForNativeFullscreenEnter {
     XCTestExpectation *expectation = [self expectationWithDescription:@"NativeFullscreenEnter"];
 
@@ -862,4 +864,114 @@ - (void) testFullScreenNative {
     [self fullScreenTestWithNative:YES];
 }
 
+- (void) testFullScreenNonNativeOptions {
+    MMAppController *app = MMAppController.sharedInstance;
+
+    // Cache test defaults
+    NSUserDefaults *ud = NSUserDefaults.standardUserDefaults;
+    NSDictionary<NSString *, id> *defaults = [ud volatileDomainForName:NSArgumentDomain];
+    NSMutableDictionary<NSString *, id> *newDefaults = [defaults mutableCopy];
+
+    // Change native full screen setting
+    newDefaults[MMNativeFullScreenKey] = @NO;
+    [ud setVolatileDomain:newDefaults forName:NSArgumentDomain];
+
+    [app openNewWindow:NewWindowClean activate:YES];
+    [self waitForVimOpenAndMessages];
+
+    MMWindowController *winController = app.keyVimController.windowController;
+    MMTextView *textView = [[winController vimView] textView];
+
+    // Test maxvert/maxhorz
+    [self sendStringToVim:@":set lines=10\n" withMods:0];
+    [self sendStringToVim:@":set columns=30\n" withMods:0];
+    [self sendStringToVim:@":set fuoptions=\n" withMods:0];
+    [self waitForVimProcess];
+
+    [self sendStringToVim:@":set fu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqual(textView.maxRows, 10);
+    XCTAssertEqual(textView.maxColumns, 30);
+    [self sendStringToVim:@":set nofu\n" withMods:0];
+    [self sendStringToVim:@":set fuoptions=maxvert\n" withMods:0];
+    [self sendStringToVim:@":set fu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertGreaterThan(textView.maxRows, 10);
+    XCTAssertEqual(textView.maxColumns, 30);
+    [self sendStringToVim:@":set nofu\n" withMods:0];
+    [self sendStringToVim:@":set fuoptions=maxhorz\n" withMods:0];
+    [self sendStringToVim:@":set fu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqual(textView.maxRows, 10);
+    XCTAssertGreaterThan(textView.maxColumns, 30);
+    [self sendStringToVim:@":set nofu\n" withMods:0];
+    [self sendStringToVim:@":set fuoptions=maxhorz,maxvert\n" withMods:0];
+    [self sendStringToVim:@":set fu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertGreaterThan(textView.maxRows, 10);
+    XCTAssertGreaterThan(textView.maxColumns, 30);
+
+    // Test background color
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xff000000]); // default is black
+
+    // Make sure changing colorscheme doesn't override the background color unlike in non-full screen mode
+    [self sendStringToVim:@":color desert\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xff000000]);
+
+    // Changing fuoptions should update the background color immediately
+    [self sendStringToVim:@":set fuoptions=background:Normal\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xff333333]);
+
+    // And switching colorscheme should also update the color as well
+    [self sendStringToVim:@":color blue\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xff000087]);
+
+    // Test parsing manual colors in both 8-digit mode (alpha is ignored) and 6-digit mode
+    [self sendStringToVim:@":set fuoptions=background:#11234567\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xff234567]);
+    [self sendStringToVim:@":set fuoptions=background:#abcdef\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithArgbInt:0xffabcdef]);
+
+    // Test setting transparency while in full screen. We always set the alpha of the background color to 0.001 when transparency is set.
+    [self sendStringToVim:@":set fuoptions=background:#ffff00\n:set transparency=50\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithRed:1 green:1 blue:0 alpha:0.001]);
+
+    [self sendStringToVim:@":set fuoptions=background:#00ff00\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithRed:0 green:1 blue:0 alpha:0.001]);
+
+    [self sendStringToVim:@":set transparency=0\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithRed:0 green:1 blue:0 alpha:1]);
+
+    // Test setting transparency outside of full screen and make sure it still works
+    [self sendStringToVim:@":set nofu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    [self sendStringToVim:@":set transparency=50 fuoptions=background:#0000ff\n" withMods:0];
+    [self sendStringToVim:@":set fu\n" withMods:0];
+    [self waitForEventHandlingAndVimProcess];
+    [self waitForEventHandlingAndVimProcess];
+    XCTAssertEqualObjects(winController.window.backgroundColor, [NSColor colorWithRed:0 green:0 blue:1 alpha:0.001]);
+
+    // Clean up
+    [[app keyVimController] sendMessage:VimShouldCloseMsgID data:nil];
+    [self waitForVimClose];
+
+    XCTAssertEqual(0, [app vimControllers].count);
+
+    // Restore settings to test defaults
+    [ud setVolatileDomain:defaults forName:NSArgumentDomain];
+}
+
 @end

src/MacVim/gui_macvim.m

@@ -633,6 +633,14 @@
 
     ASLogDebug(@"back=%ld norm=%ld", gui.def_back_pixel, gui.def_norm_pixel);
 
+    // If using a highlight group for fullscreen background color we need to
+    // update the app when a new color scheme has been picked. This function
+    // technically wouldn't be called if a user manually set the relevant
+    // highlight group to another color but works in most use cases when they
+    // just change the color scheme.
+    if (fuoptions_flags & FUOPT_BGCOLOR_HLGROUP)
+        gui_mch_fuopt_update();
+
     [[MMBackend sharedInstance]
         setDefaultColorsBackground:gui.def_back_pixel
                         foreground:gui.def_norm_pixel];

src/optionstr.c

@@ -4839,16 +4839,16 @@ check_fuoptions(void)
 		return FAIL;
 	    if (p[i] == '#')
 	    {
-		// explicit color (#aarrggbb)
+		// explicit color (#aarrggbb or #rrggbb)
 		i++;
 		for (j = i; j < i + 8 && vim_isxdigit(p[j]); ++j)
 		    ;
-		if (j < i + 8)
-		    return FAIL;    // less than 8 digits
+		if (j != (i + 8) && j != (i + 6))
+		    return FAIL; // has to be either 6 or 8 digits
 		if (p[j] != NUL && p[j] != ',')
 		    return FAIL;
 		new_fuoptions_bgcolor = 0;
-		for (k = 0; k < 8; ++k)
+		for (k = 0; k < (j - i); ++k)
 		    new_fuoptions_bgcolor = new_fuoptions_bgcolor * 16
 							   + hex2nr(p[i + k]);
 		i = j;