Create Custom Ribbon and Open an Application page in Model Popup using that custom Ribbon in SharePoint 2010

<CustomAction

   Id="DemoHelloWorldButton"

   RegistrationType="List"

   RegistrationId="101"

   Location="CommandUI.Ribbon">

    <CommandUIExtension>

      <CommandUIDefinitions>

        <CommandUIDefinition

         Location="Ribbon.Documents.New.Controls._children">

          <Button

           Id="Ribbon.Documents.New.Controls.DemoHelloWorldButton"

           Alt="My own Ribbon"

           Sequence="10"

           Image32by32="/_LAYOUTS/imagesCANGKL7U.jpg"

           Command="Demo_HelloWorld"

           LabelText="My Own Ribbon"

           TemplateAlias="o2"/>

        </CommandUIDefinition>

      </CommandUIDefinitions>

      <CommandUIHandlers>

        <CommandUIHandler Command="Demo_HelloWorld"

          CommandAction="javascript:

          var dialogOptions = {

          url:'/_layouts/Testfolder/Test.aspx',

          title: 'Add Items',

          allowMaximize: true,

          showClose: true,

          width:900,

          height:900

        };

        SP.UI.ModalDialog.showModalDialog(dialogOptions); " />

       

      </CommandUIHandlers>

    </CommandUIExtension>

    </CustomAction>